/*
  This version of HLRC protocol was modified in order to use
  the TCP/IP protocol instead of VIA protocol.

  Rafael Mendes da Silva, Marcelo Lobosco, Lauro Whately, Claudio Amorim

  Parallel Computing Laboratory
  UFRJ/COPPE - Systems and Computing Eng.

  Support: rmendes@cos.ufrj.br

*/

/*
  ORIGINAL MESSAGE:

  Home-Based Lazy Release Consistency (HLRC)
  Distributed Shared Memory protocol for PC Clusters
  using the Virtual Interface Architecture(VIA)

  Liviu Iftode, Murali Rangarajan 
  Copyright (C) 2000 Rutgers University

  hlrc.c - source code for the HLRC protocol

  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  "Software"), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:
 
  The above copyright notice and this permission notice shall be
  included in all copies or substantial portions of the Software.
  THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  IN NO EVENT SHALL RUTGERS UNIVERSITY BE LIABLE FOR ANY CLAIM, DAMAGES 
  OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 
  OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR 
  THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
  Except as contained in this notice, the name of Rutgers University
  or the contributors shall not be used in advertising or otherwise to
  promote the sale, use or other dealings in this Software without prior
  written authorization.

  The first HLRC protocol was proposed for the SHRIMP project at Princeton
  University.
*/

#include <stdio.h>
#include <assert.h>
#include <stdlib.h>
#include <math.h>
#include <memory.h>
#include <sys/mman.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <sys/socket.h>
#include <fcntl.h>
#include <unistd.h>
#include <signal.h>
#include <stdarg.h>
#include <netdb.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include <errno.h>
#include <semaphore.h>
#include <sys/uio.h>
#include <ctype.h>
#include <ucontext.h>

#define COMP_ERR HLRC_ErrPrint("%d %d %d %d %d %d %d %d %d %d %d %d %d \n",\
	 EBADF,ENOTSOCK,ENOTCONN,EFAULT,EMSGSIZE,EAGAIN,EWOULDBLOCK,ENOBUFS,\
	 EINTR,ENOMEM,EINVAL,EPIPE,errno);

//#define HLRC_DEBUG

#define HLRC_TCP

//#define _REENTRANT

#ifdef PROC_BIND_SUPPORT
#include <sys/sysmp.h>
#endif

#include <pthread.h>

#define DESCRIPTOR_ALIGNMENT 64

// we have used VIP_DESCRIPTOR_ALIGNMENT up to now, the value is the same.

#if 0
#define LOSE_TTY
#endif

#if 0
#define INDIRECT_DIFFS
#endif

#if !defined(COPY_ON_RECV) || !defined(COPY_ON_SEND)
#define PROT_HANDLE
#endif

#define MAX(x,y) (x>y)?x:y

#if !defined(COPY_ON_RECV) && !defined(COPY_ON_SEND)
#define DIRECT_DIFFS
#endif
/* 
   Host file used to spawn processes for the parallel phase 
   Each line in the host file contains a hostname and a 
   discriminator(used by VIA) for the host.
   Caveat: The VIA implementation doesn't seem to support 
   full hostnames(hostname + domain). The host file which is 
   used to specify the nodes participating in the protocol 
   should contain only short hostnames. Currently, the host
   file should have the same number of hosts as required by
   the application.
*/
#define HOSTS_FILE        "hlrc_host_file"


#ifdef PROT_HANDLE
/* 
   mmap is used to generate two handles to the shared memory
   region. One handle is used by the protocol to access/manipulate
   shared memory without any protection violations. The other 
   handle is used by the application. A backing file of the size
   of the shared memory region is required with mmap, as mandated
   by the MAP_SHARED option to mmap.
*/
#define MMAP_BACKING_FILE "/var/tmp/hlrc_mmap_file"
#endif


/* 
   Log file used by the DSM protocol. When run in the DEBUG mode,
   all the protocol events are logged in this file.
*/
#define LOG_FILE          "/var/tmp/hlrc_log_file"

/*
  Extension for the file name where the performance statistics
  are printed out. All the statistics are generated in a file
  with the application pathname suffixed by STAT_FILE_EXT.
*/
#define STAT_FILE_EXT     ".stat"

/*
  Macro to read the value of the processor cycle counter. The 
  processor cycle counter can be read from a special register
  on the Pentium II processors. The use of this macro implicitly
  assumes that Pentium II processor is being used.
*/
#define GETTIME(x)            asm volatile (".byte 0x0f, 0x31" : "=A" (x));

/*
  Processor speed in KHz. This is defined in c.m4.hlrc(which
  defines the macros used by the application) as well. Change 
  this value in both files to the speed of the processor used.
*/
double CPU_MHZ;

/*
  Topmost location in the buffer used to send and receive
  control messages for PAGEs and LOCKs
*/
#define HLRC_MSG_HDR_TOP  1022

/*
  Length of the control message header
*/
#define MSG_HDR_LEN       3

#define WORDSIZE         4

/*
  Starting location in the buffer used for DIFFs
*/
#define HLRC_DIFF_HDR_START(bufId) ((bufId) * sizeof(TDiffControl) / WORDSIZE)

/*
  Macro to build a header from various fields - 
  type  : Type of the message
  sender: Sender of the message
  bufId : Id of the buffer used(to support some sort of windowing) 
  node  : Origin of the message(if the message has been forwarded) 
  This header is sent as immediate data in an RDMA write 
*/
#define MESG_HDR(type, sender, bufId, node) ((node << 24) | (bufId << 20) | (sender << 16) | type)

/*
  Extract various fields from the header which is sent as 
  immediate data in an RDMA write
*/
#define MESG_TYPE(hdr)  ((hdr) & 0x0000ffff)
#define SENDER_ID(hdr)  (((hdr) & 0x000f0000) >> 16)
#define BUFFER_ID(hdr)  (((hdr) & 0x00f00000) >> 20)
#define NODE_ID(hdr)    (((hdr) & 0xff000000) >> 24)

/*
  Size(number of pages) of shared memory used by the application, 
  The entire shared memory is pre-allocated in the initialization 
  phase. This parameter has to be changed to suit application 
  requirements.
*/
#if defined(COPY_ON_SEND) && defined(COPY_ON_RECV)

#define PAGE_NUM  (130*1024)	/*lobosco: antes 130*1024, max 100 */

#else

/*
  To avoid copies, the entire shared memory has to be registered
  with VIA. Currently, the amount of shared memory which can be
  registered with VIA is restricted to 128MB(32K pages) by the 
  VIA implementation.
*/
#define PAGE_NUM  (130*1024)	/*lobosco: antes 28*1024; max 100 */

#endif

/*
  Maximum number of locks available to the application. If 
  you change here, please change in c.m4.hlrc as well.
*/
#define NLOCKS    (3*1024*1024)

/*
  Size of the buffer(bin) used to store write notices for the 
  nodes participating in the DSM protocol. These buffers are
  cleared at barrier time.
*/
#define BINSIZE   (3*1024*1024)	//PAGE_NUM

/*
  Number of dirty pages which can be present at any node
*/
#define NUPDATES  PAGE_NUM

/*
  Number of dirty pages which can be maintained at a node, which
  is not the home for any of these pages. This limits the number 
  of twins(unmodified copies fetched from home) at a node. The
  dirty list is cleared at the end of every interval.
*/
#define MAX_TWINS   PAGE_NUM


//#define WORDSIZE         4

/*
  An upper limit on the number of processors to be used for the 
  DSM protocol.
*/
#define NPROCS           16	/*lobosco: antes 32 */

#define HLRC_PROT_NONE   0
#define HLRC_PROT_READ   1
#define HLRC_PROT_RW     3
#define HLRC_EXCL_STATE  4

#ifdef NO_SWAP_READONLY_PAGE

#define HLRC_PROT_READ_DISCARD_AT_SWAP 8
#define QUERY_PAGE_PROT 16
#define PAGE_NOT_PRESENT 126

#endif


/*
  Different types of request messages. These are messages
  which require a response(at the origin of the message).
  These are sent as RDMA Write(with immediate data) messages
  in VIA.

  PAGE:   Request message for a page
  LOCK:   Request message for a lock
  DIFF:   Message for diff
*/
#define PAGE            16  //antes: 16
#define LOCK            32  //antes: 32
#define DIFF            64  //antes: 64

#define WAIT           128
#define TIMEDOUT       256
#define NOSLEEP        512
#define SIGNAL        1024
#define BROADCAST     2048


/*
  Messages sent as RDMA Writes in VIA without a receive operation
  at the remote node.       
  DIFF_MESG    : To send the DIFFs of dirty pages to their homes 
  PAGE_MESG    : To send the requested page from the home
  BIN_MESG     : To send the write notices at the barrier and when
  servicing request for a lock
  BARRIER_MESG : To send the control message for the barrier

  DIFFACK_MESG : To send the ack for a DIFF to enable further 
  sending of DIFFs from the source
  REQUEST_MESG : Control message for responses to requests for 
  pages and locks

  BARR_END_MESG: Message to signal completion of barrier at a node
  SVM_ADDR_MESG: Each node exports the local shared memory region
  by broadcasting a pointer to the region
  START_ADDR_MESG: Message to synchronize the starting addresses
  of the shared memory regions on different nodes
*/

#define DIFF_MESG        1
#define PAGE_MESG        3
#define BIN_MESG         5
#define DIFFACK_MESG     2
#define REQUEST_MESG     4
#define BARRIER_MESG     6
#define BARR_END_MESG    7
#define START_ADDR_MESG  8
#define SVM_ADDR_MESG    9
#define END_MESG        10

/*
  Static id's for the application thread and the communication 
  thread. Just to avoid calling pthread_self() too many times 
  in the critical path.
*/
#define APP_THREAD       0
#define SERVER_THREAD    1

#define VIA_DESC_SIZE   64

/*
  Sizes of the completion queues to send and receive VIA messages.
*/
#define SENDCQ_SIZE      (4*NPROCS)
#define RECVCQ_SIZE      (4*NPROCS)

#ifdef PROT_HANDLE
#define MAX_REG_PART 8		// used to be 4, vsc
#define MAX_REGMEM_PAGES (20*1024)
#endif

/*
  Windowing can be used to potentially reduce the latency of 
  certain operations. Not used currently.
*/
#define MAX_SEND_WINDOW  1

/*
  Maximum number of messages that can be simultaneously
  received on a VI.
*/
#define MAX_SIMUL_MESG   8

/*
  Sizes of windows used for DIFF and BARRIER messages.
*/
#define DIFF_WIN_SIZE    2

#define DIFF_MAX_SEND    8

#define BARR_WIN_SIZE    2

/*
  Port to be used for the UDP socket barrier.
*/
#define SOCK_BARR_PORT  11009
#define BARR_SOCK_MESG  "UDP_BARR"

typedef struct recv_s recv_t;

struct recv_s{
  int nodeId;
  int bufId;
  int mesgType;
  int senderId;
  int index;
};

// Reserved to TCP/UDP related declarations

#define BASE_SERV_PORT 11500	// initial constant to calculate the port to handle the requests
#define BASE_RDMA_SERV_PORT (BASE_SERV_PORT + 1000) // used in RDMA requests
#define TRANS_RETURN int
#define TRANS_SUCCESS 0
#define TRANS_ERROR  -1
#define TRANS_NUM_FIELDS 4
#define BUFFER_PKG_SIZE 1800000 // antigo 100000

typedef struct sockaddr SA;

typedef struct _TRANS_data
{
  int length;
  char *addr;
}
TRANS_data;

typedef struct _TRANS_msg
{
  int header;
  TRANS_data local, remote;
}
TRANS_msg;

typedef struct _TRANS_msg_mapp
{
  int type;
  int size;
  void *addr;
}
TRANS_msg_mapp;

char buffer_pkg_recv[BUFFER_PKG_SIZE];
char buffer_pkg_send[BUFFER_PKG_SIZE];

sem_t sem_barrier, mutex_comm, mutex_barrier, sem_lock, sem_page, sem_barr_end, mutex_barr_end;

struct sockaddr_in appAddr;	// used for bind appSocket
struct sockaddr_in servAddr;	// used for bind servSocket
struct sockaddr_in rdmaAppAddr; // userd for bind rdmaServSock
struct sockaddr_in rdmaServAddr; // used for bind rdmaServSock
struct in_addr otherInAddr[NPROCS];	// remote hosts IP

int fila[NPROCS];
short in_fila, fim_fila, tam_fila;
int appSock[NPROCS];		// local endpoint for the app thread(used to send requests too)
int servSock[NPROCS];		// local endpoint for the server thread
int accSock[NPROCS];       // utilized to estabilish the connection
int max_sock = 0;  // socket used in select

int rdmaAppSock[NPROCS];
int rdmaServSock[NPROCS];
int rdmaAccSock[NPROCS];
int rdma_max_sock = 0; // used in rdma select

fd_set req_socks;		// set of descriptors to use in select
fd_set rdma_socks;   // used in rdma select

void TRANS_init_comm ();
TRANS_RETURN TRANS_recv_rdma_mapp (TRANS_msg_mapp * msg, int from);
TRANS_RETURN TRANS_send_rdma_mapp (TRANS_msg_mapp * msg, int to);
TRANS_RETURN TRANS_send_req (TRANS_msg * msg, int to);	// send a request

TRANS_RETURN TRANS_send_req_ctrl (TRANS_msg * msg, TRANS_msg * msg_ctrl, int to);
// send a request with control message

TRANS_RETURN TRANS_wait_req (TRANS_msg * msg, TRANS_msg * msg_ctrl, int from);
// wait for a request (Server Thread)

TRANS_RETURN TRANS_send(int to, char *msg, int size); // envia mensagem (sem scatter-gather)

TRANS_RETURN TRANS_recv(int from, char *msg, int size); // recebe mensagem (sem scatter-gather)

TRANS_RETURN TRANS_recv_gather(int from, struct iovec * vector, int size); //utiliza scatter-gather

TRANS_RETURN TRANS_send_scatter(int to, struct iovec * vector, int size); // utiliza scatter-gather

// end of TCP/UDP declarations

/*
  Data structure to store information about all shared
  memory pages.
*/

typedef struct _per_page_struct
{
  short size;
  int *twin;			/* unmodified copy(from home) */
#ifdef INDIRECT_DIFFS
  int state;
  int dirty_and_invalid;
#else
  int state;
#endif
  short home;
  unsigned my_stamp[NPROCS + MSG_HDR_LEN];	/* version of the page        */
}
TPerPage;

/*
  Data structure to store information about locks used in
  the DSM protocol.
*/
typedef struct _lock_data_struct
{
  int state;
  int last;			/* last owner of the lock              */
  int stamp[NPROCS];		/* global time stamp vector at release */
  int index[NPROCS];		/* index into bin buffer already seen  */
  int counter;
  int owner;
  int wait[NPROCS];		/* processor waiting a signal for this lock */
}
TLockData;

/*
  Data structure for sending control info with a DIFF message.
*/
typedef struct _diff_control_struct
{
  int size;			/* size of the DIFF                    */
  int tmst;			/* version of page the DIFF brings in  */
  int no;			/* buffer id in the DIFF window        */
  int firstsize;		/* size of first DIFF segment          */
  int page;			/* page for which DIFF is sent         */
  int proc;			/* node from which DIFF is sent        */
  int cmd;			/* type of the message: DIFF           */
}
TDiffControl;

/*
  Data structure for storing pending requests.
*/
typedef struct _pending_struct
{
  int type;			/* type of pending request(PAGE/LOCK)  */
  int addr;			/* id of requested item(page/lock num  */
  unsigned stamp[NPROCS];	/* requested version                   */
}
TPending;

/*
  Data structure for the DIFF buffer windowing.
*/
typedef struct hlrc_protocol_control_struct
{
  int diffBufferFree[NPROCS][DIFF_WIN_SIZE];
}
TProtocolControl;

/*
  Data structure used for control information in a BARRIER message.
*/
typedef struct _bar_buff_struct
{
  int buf[NPROCS][2];
}
TBarrierBuff;

/*
  Data structure to collect various pieces of statistics
  information.
*/
typedef struct HLRC_stat_struct
{
  char abspathstatfile[1024];	/* absolute pathname to stats file     */

  float time_exec;		/* execution time                      */
  float time_comm;		/* time spent waiting to fetch pages   */
  float time_lock;		/* time spent waiting to acquire locks */
  float time_barr;		/* time spent in barriers              */
  float time_over;		/* time spent in protocol actions      */
  float time_comp;		/* time spent in computation(appln)    */
  float time_handler;		/* time spent in the handler           */

  float time_spinning;
  float diffwait;

  unsigned page_faults;
  unsigned page_misses;
  unsigned diff_created;
  unsigned diff_applied;
  unsigned homes;

  unsigned lock_acq;
  unsigned lock_remote;
  unsigned barriers;
  unsigned msg_control;
  unsigned msg_data;
  unsigned traffic_control;
  unsigned traffic_barr;
  unsigned traffic_data;
  unsigned traffic_diff;
  unsigned requests_recv;
  unsigned allocated;
  unsigned forward;
  unsigned num_swaps;
}
THLRCStat;

/*
  Data structures to send and receive DMA mappings to be
  used for RDMA messages.
*/
typedef struct local_memory_struct
{
  void *pAddress;
}
TLocalMemory;

typedef struct remote_memory_struct
{
  void* pAddress;
}
TRemoteMemory;

/*
 * Variaveis usadas pela bloquear a thread da aplicacao
 */

pthread_mutex_t mutex_ = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t cond_   = PTHREAD_COND_INITIALIZER; 
pthread_mutex_t lmutex_ = PTHREAD_MUTEX_INITIALIZER;
pthread_cond_t lcond_   = PTHREAD_COND_INITIALIZER; 



/* 
   Data structures used by HLRC internally 
*/

int HLRC_myself=-1;
int HLRC_nhosts;
int HLRC_nprocs;
int HLRC_init_flag;
int HLRC_clock_stopped;
int HLRC_reset_flag;

/*
  Actual size(number of pages) of shared memory used by the 
  application.
*/
int HLRC_total_pages;
/*
  Actual number of locks used by the application.
*/
unsigned HLRC_nlocks;

/*
  Hostname of local node. The VIA library seems to process 
  correctly SHORT hostnames ONLY.
*/
char HLRC_myname[1024];
char HLRC_myname_short[1024];

/*
  Hostnames of all nodes participating in the protocol.
*/
char *HLRC_hostnames[NPROCS];

struct timeval HLRC_internal_time;
unsigned HLRC_start;
char HLRC_program[1024];

/*
  Mutexes used to coordinate access to shared data between
  the communication thread and the application.
*/
pthread_mutex_t HLRC_critical_section_mutex[2];

unsigned long long HLRC_begin, HLRC_final;

pthread_t app_thread, srv_thread, srv_rdma_thread;
pid_t app_pid;
int oldprio;
int AppInsideCS;

/*
  msg_type_index:   ptr to mesg type in the request mesg buffer
  msg_node_index:   ptr to mesg origin in the request mesg buffer
  - different from sender for forwarded messages
  msg_item_index:   ptr to id of item(page/lock num) being requested
  msg_tmst1_start:  ptr to start of version(timestamp) of item requested
  msg_tmst2_start:  ptr to start of second timestamp(when sending locks)
*/
int msg_type_index;
int msg_node_index;
int msg_item_index;
int msg_tmst1_start;
int msg_tmst2_start;

int pagesize;
int logpagesize;
int diff_buff_size;

/*
  Starting address of shared memory region used by application.
*/
int *_first_alloc, *_last_alloc;

int _size_adjust;
unsigned _first_shared_page=0;
unsigned _last_shared_page=0;
unsigned _nopage;

#ifdef PROT_HANDLE
int mmap_fd;
unsigned prot_first_shared_page;
#endif

/*
  Number of twins used at any instant of time.
*/
volatile unsigned int _notwins;
volatile int *_free_twins[PAGE_NUM];

struct sigaction HLRC_sig;
struct sigaction HLRC_sig5;

/*
  Id of next buffer(from window) to use to send DIFFs.
*/
volatile int _next_diff[NPROCS];

/*
  Array of pending requests(maximum of one per remote node).
*/
volatile TPending _pending[NPROCS];


/*
  Version of page flushed to home
*/
volatile int *_home_flush[NPROCS];

/*
  List of dirty pages currently at the node.
*/
volatile int _update_list[NUPDATES];
volatile int _index_update=0;
volatile int _index_bin[NPROCS];


/*
  Snapshot of what has been seen at the node.
*/
volatile unsigned _last_stamp[NPROCS];

char *HLRC_stats_buff[NPROCS];

int _bar_count;
THLRCStat HLRC_stat;

/*
  Variables for various parameters to be specified
  to the HLRC protocol.
*/
char *hosts_file;
char *nic_name;
char *mmap_backing_file;
char *log_file;
char *sock_barr_port_str;
int sock_barr_port;

FILE *logfile;


#ifdef PROC_BIND_SUPPORT

unsigned int num_cpu;
unsigned int app_cpu, handler_cpu, curr_cpu;

#endif


/* 
   Protocol data structures which are registered
   with VIA to be used as source or destination
   buffers for messages 
*/

#ifdef PROT_HANDLE
/*
  Handle to shared memory used by the protocol. The
  protocol can operate on shared memory without any 
  risk of protection violations.
*/
int *prot_first_alloc;

int num_reg_part;

#endif

volatile TPerPage *_per_page;

volatile TProtocolControl *hlrc_protocol_control;

volatile TDiffControl *_diff_control[NPROCS];

volatile TLockData *_lock_data;

/*
  Global timestamp vector.
*/

volatile int *_global_stamp;

/*
  Holds write notices of updates(seen at the local node) which 
  happened in intervals dating back to the last barrier.
*/

volatile int *_bin[NPROCS];

/*
  Buffer used to send control info with the BARRIER message.
*/

int *buf[NPROCS];

/*
  Buffer used to acknowledge the DIFF message.
*/

int *diffAck;

/*
  Buffer to send request(PAGE/LOCK/DIFF) messages.
*/

volatile int *HLRC_request_buffer[NPROCS];

/*
  Buffers used to send responses and to forward messages.
*/

volatile int *HLRC_scratch_buffer[NPROCS][NPROCS];

#ifdef COPY_ON_RECV
/*
  Buffer used to receive the page if it is not possible
  to register the entire shared memory and thus avoid
  the copy.
*/
volatile int *HLRC_page_recv_buff;
//VIP_MEM_HANDLE hPageRecvBuffHandle;

#endif


/*
  Buffer used to send BARRIER messages.
*/

volatile TBarrierBuff *_bar_buff;

/*
  Buffer used to signal end of barrier.
*/

volatile int *_bar_end_buff;

int prev_barr_complete;

#ifdef COPY_ON_SEND
/*
  If unable to register entire shared memory, a segment in
  shared memory cannot be used as the source buffer in VIA
  sends. Register small send buffers to which corresponding 
  shared memory is copied and pass these buffers to the 
  VIA library routines.
*/
volatile int *HLRC_page_send_buff[NPROCS];

#endif


/*
  Buffers to send DIFFs.
*/
volatile int *_diff_buff_send[NPROCS];

/*
  Buffers to receive DIFFs.
*/
volatile int *_diff_buff_recv[NPROCS];

int *start_addr;

/* 
   Data structures for the UDP socket barrier 
*/
int barr_sock;
struct sockaddr_in remote_sock[NPROCS];
char sendline[128];


/* 
   Handle for the communication thread 
*/
pthread_t hServerThreadHandle;
pthread_t hServerThreadRDMAHandle;

/*
  Flags to mark the completion of sends from the 
  application thread.
*/
volatile int cSendIncomplete[NPROCS];
/*
  Flags to mark the completion of sends from the 
  communication thread.
*/
volatile int sSendIncomplete[NPROCS];

#ifdef DIRECT_DIFFS
volatile int cDiffSendIncomplete[NPROCS][DIFF_MAX_SEND];
#endif

/*
  Discriminators can be used to uniquely identify nodes
  at connection setup. Currently, these discriminators
  are statically assigned and read from the host file
  during initialization.
*/
int VIA_discriminators[NPROCS];
long lLocalDiscriminatorId = 1000;

volatile int mesgSendIndex[NPROCS];

/* 
   Data structures to store the mappings for RDMA writes 
*/

int *pRemoteStartAddr[NPROCS];

#ifdef PROT_HANDLE
TRemoteMemory *pRemoteMemory[NPROCS][MAX_REG_PART];

#endif

int *pRecvBuffs[NPROCS][MAX_SIMUL_MESG];

TLocalMemory *pRDMA;

#ifdef COPY_ON_RECV
TRemoteMemory *pPageBuffs[NPROCS];
#endif

TRemoteMemory *pProtocolControls[NPROCS];

TRemoteMemory *pBarrierBuffs[NPROCS];

TRemoteMemory *pBarrEndBuffs[NPROCS];

TRemoteMemory *pRequestBuffs[NPROCS][NPROCS];

TRemoteMemory *pRemoteBins[NPROCS][NPROCS];

TRemoteMemory *pDiffBuffs[NPROCS];

void _apply_wn (int bindex, int tmst, int i);
void _compute_and_send_diff (int relpage, int *new, int size);
void HLRC_PageHandler5 (int sig, siginfo_t *sip, void *uc);
void HLRC_PageHandler (int sig, siginfo_t *sip, void *uc);
void HLRC_SigioHandler (int dummy, struct sigcontext_struct scs);
void HLRC_Barrier ();

/* ---------------------------- helper functions --------------------*/
int
cpu_frequency () {
  FILE    *fp;
  char    buf[128];
  double  val;
  int     ret = 0;

  if ((fp = fopen ("/proc/cpuinfo", "r")) != NULL)
    {
      while (fgets (buf, sizeof (buf), fp) != NULL)
	{
	  if (sscanf (buf, "cpu MHz  : %lf\n", &val) == 1)
	    {
	      CPU_MHZ = val * 1000;
	      ret = 1;
	      break;
	    }
	}
      fclose (fp);
    }
  return ret;
}

/*
 * 	Get the number of the host eg: cl2 -> 2
 */

int
HLRC_HostNumber(char *hostname)
{
  int idx1;
  idx1 = 0;
  while (!isdigit(hostname[idx1]))
    idx1++;
  return atoi(hostname + idx1);
}	

/*
  Get the short version of the hostname from the full hostname.
*/
void
HLRC_HostShortName (char *hostname)
{
  int i = 0;
  int len = strlen (hostname);

  while ((i < len) && (hostname[i] != '.'))
    {				//lobosco: antes '\.'
      i++;
    }

  for (; i < len; i++)
    {
      hostname[i] = 0;
    }
 
}


/*
  Get the full hostname from the given hostname.      
*/
void
HLRC_HostFullName (char *Host)
{
  struct hostent *he = gethostbyname (Host);

  strcpy (Host, he->h_name);
}


/*
  Get the number of bits(log2) to represent pagesize
*/
int
getlogpagesize (int pgsize)
{
  int i = 0;

  while (++i)
    {
      if (pgsize < pow (2, i))
	return (i - 1);
    }
  return 0;
}


/*
  Print a message on stderr with the hostname as a prefix
*/
void
HLRC_ErrPrint (char *fmt, ...)
{
  va_list args;

  fprintf (stderr, "[%s] ", HLRC_myname_short);

  va_start (args, fmt);
  (void) vfprintf (stderr, fmt, args);
  va_end (args);

  fflush (stderr);

}


/*
  Get the absolute pathname for a specified file
*/
void
HLRC_AbsoluteCurrPath (char *fileName, char *absFileName)
{
  char *res;
  char tmp[1024];

  if (*fileName == '/' && *(fileName + 1) == '/')
    {
      strcpy (absFileName, fileName);
      return;
    }

  res = getcwd (absFileName, 1024);
  if (res == NULL)
    {
      HLRC_ErrPrint ("Cannot get current working dir");
      exit (1);
    }

  strcpy (tmp, "/");
  strcat (tmp, absFileName);
  strcpy (absFileName, tmp);
  strcat (absFileName, "/");
  strcat (absFileName, fileName);
}


/*
  Read the hosts file and build a list of hosts participating
  in the protocol
*/
void
HLRC_Hosts (char *file, int *nhosts)
{
#define MAXHOSTFILELINE 128

  FILE *f;
  char line[MAXHOSTFILELINE];
  int n;

  if (!(f = fopen (file, "r")))
    {
      fprintf (stderr, "Cannot open hostfile %s\n", file);
      fprintf (stderr,
	       "Please check the parameter HOSTS_FILE in the source\n");
      exit (1);
    }

  *nhosts = 0;

  while ((n = fscanf (f, "%[^\n]\n", line)) != EOF)
    {
      if (n == 0)
	{
	  fprintf (stderr, "format error in hostfile %s", file);
	  exit (1);
	}
      else if (line[0] == '#')
	{
	  /* comment line, ignore */
	}
      else
	{
	  if (*nhosts == NPROCS)
	    {
	      fprintf (stderr, "Too many hosts in partition file\n");
	      exit (1);
	    }

	  HLRC_hostnames[*nhosts] = (char *) malloc (MAXHOSTFILELINE);
	  sscanf (line, "%s %d", HLRC_hostnames[*nhosts],
		  &VIA_discriminators[*nhosts]);
	  (*nhosts)++;
	}
    }
  fclose (f);

}


/*
  Print a line of formatted stats in a shared buffer
*/
void
stat_sprintf (char *fmt, ...)
{
  char mybuf[pagesize];
  va_list args;

  va_start (args, fmt);
  (void) vsnprintf (mybuf, 4096, fmt, args);
  va_end (args);

  strncat (HLRC_stats_buff[HLRC_myself], mybuf, pagesize);
}

/*
  Print all the statistics about the application performance
  in a shared buffer
*/
void
HLRC_PrintStatsInBuffer ()
{
  long Time;

  HLRC_stat.time_comp = HLRC_stat.time_exec -
    (HLRC_stat.time_comm + HLRC_stat.time_lock +
     HLRC_stat.time_barr + HLRC_stat.time_over) / CPU_MHZ;
  Time = time (0);

  stat_sprintf ("\n\nStatistics for %s on node %d (%f MHz) at %s\n",
		HLRC_program, HLRC_myself, CPU_MHZ, ctime (&Time));

  if (HLRC_myself == 0)
    {
      stat_sprintf ("Used %d pages of shared memory\n", HLRC_stat.allocated);
    }

  stat_sprintf ("HLRC options: ");

#ifdef COPY_ON_RECV
  stat_sprintf ("COPY_ON_RECV, ");
#else
  stat_sprintf ("NO COPY_ON_RECV, ");
#endif

#ifdef COPY_ON_SEND
  stat_sprintf ("COPY_ON_SEND, ");
#else
  stat_sprintf ("NO COPY_ON_SEND, ");
#endif

#ifdef HLRC_DEBUG
  stat_sprintf ("HLRC_DEBUG, ");
#else
  stat_sprintf ("NO HLRC_DEBUG, ");
#endif

  if (HLRC_reset_flag)
    {
      stat_sprintf ("statistics reset.\n");
    }
  else
    {
      stat_sprintf ("*** statistics NOT reset ***\n");
    }

  stat_sprintf ("\n");

  stat_sprintf ("%-28s: %d nodes\n", "System Configuration", HLRC_nprocs);

  stat_sprintf ("%-28s: %.2f(msec)\n", "Execution time", HLRC_stat.time_exec);
  stat_sprintf ("%-28s: %.2f(msec)\n", "Computation time",
		HLRC_stat.time_comp);
  stat_sprintf ("%-28s: %.2f(msec)\n", "Page time",
		HLRC_stat.time_comm / CPU_MHZ);
  stat_sprintf ("%-28s: %.2f(msec)\n", "Lock time",
		HLRC_stat.time_lock / CPU_MHZ);
  stat_sprintf ("%-28s: %.2f(msec)\n", "Barrier time",
		HLRC_stat.time_barr / CPU_MHZ);
  stat_sprintf ("%-28s: %.2f(msec)\n", "Overhead time",
		HLRC_stat.time_over / CPU_MHZ);
  stat_sprintf ("%-28s: %.2f(msec)\n", "Handler time",
		HLRC_stat.time_handler / CPU_MHZ);
  stat_sprintf ("\n");

  stat_sprintf ("%-28s: %d\n", "Page faults", HLRC_stat.page_faults);
  stat_sprintf ("%-28s: %d\n", "Page misses", HLRC_stat.page_misses);
  stat_sprintf ("%-28s: %d\n", "Lock acquired", HLRC_stat.lock_acq);
  stat_sprintf ("%-28s: %d\n", "Lock misses", HLRC_stat.lock_remote);
  stat_sprintf ("%-28s: %d\n", "Diff created", HLRC_stat.diff_created);
  stat_sprintf ("%-28s: %d\n", "Diff applied", HLRC_stat.diff_applied);
  stat_sprintf ("%-28s: %d\n", "Barriers", HLRC_stat.barriers);
  stat_sprintf ("\n");

  stat_sprintf ("%-28s: %d\n", "Number of control messages",
		HLRC_stat.msg_control);
  stat_sprintf ("%-28s: %d\n", "Number of data messages", HLRC_stat.msg_data);
  stat_sprintf ("%-28s: %d(bytes)\n", "Control traffic",
		HLRC_stat.traffic_control);
  stat_sprintf ("%-28s: %d(bytes)\n", "Data traffic", HLRC_stat.traffic_data);
  stat_sprintf ("%-28s: %d(bytes)\n", "Diff traffic", HLRC_stat.traffic_diff);
  stat_sprintf ("%-28s: %d(bytes)\n", "Barrier traffic",
		HLRC_stat.traffic_barr);
  stat_sprintf ("%-28s: %d\n", "Requests received", HLRC_stat.requests_recv);
  stat_sprintf ("%-28s: %d(pages)\n", "Application memory",
		HLRC_stat.allocated);
  stat_sprintf ("%-28s: %d\n", "Number of swaps", HLRC_stat.num_swaps);
  stat_sprintf ("%-28s: %d(pages)\n", "Home Pages", HLRC_stat.homes);
  stat_sprintf ("%-28s: %d\n", "Forwarded page requests", HLRC_stat.forward);
  stat_sprintf ("\n");

  stat_sprintf ("Time waiting for diff buffers: %.2f msec\n",
		HLRC_stat.diffwait / CPU_MHZ);
  stat_sprintf ("Time waiting for updates at home: %.2f msec\n",
		HLRC_stat.time_spinning / CPU_MHZ);
}


/*
  Create a UDP socket for a socket barrier used for 
  synchronization among participating nodes in the 
  initialization phase. An asymmetric master-slave
  barrier is used, where node 0 is chosen to be the
  "master" and all the other nodes are "slaves". At
  a barrier, all the slave nodes send a message to 
  the master, which responds after it receives a 
  barrier message from all the other nodes.
*/
void
CreateBarrierSocket ()
{
  int i;
  int reuse_opt = 1;
  struct sockaddr_in serv_addr;
  struct hostent *phostent;


  if ((barr_sock = socket (AF_INET, SOCK_DGRAM, 0)) < 0)
    {
      HLRC_ErrPrint ("Can't open barrier socket\n");
      exit (1);
    }


  bzero ((char *) &serv_addr, sizeof (serv_addr));
  serv_addr.sin_family = AF_INET;
  serv_addr.sin_addr.s_addr = htonl (INADDR_ANY);
  serv_addr.sin_port = htons (sock_barr_port);
  i = bind (barr_sock, (struct sockaddr *) &serv_addr, sizeof (serv_addr));
  if (i < 0)
    {
      HLRC_ErrPrint ("Can't bind local address");
      exit (1);
    }

  if (setsockopt (barr_sock, SOL_SOCKET, SO_REUSEADDR,
		  (void *) &reuse_opt, sizeof (reuse_opt)) < 0)
    {
      HLRC_ErrPrint ("setsockopt error\n");
      perror ("");
      exit (1);
    }

  for (i = 0; i < HLRC_nprocs; i++)
    {
      if (i == HLRC_myself)
	{
	  continue;
	}

      remote_sock[i].sin_family = AF_INET;
      if ((phostent =
	   gethostbyname (HLRC_hostnames[i])) == (struct hostent *) NULL)
	{
	  HLRC_ErrPrint ("Host %s not found.\n", HLRC_hostnames[i]);
	  exit (1);
	}
      else
	{
	  struct in_addr in;
	  memcpy (&in.s_addr, phostent->h_addr_list[0], sizeof (in.s_addr));
	  remote_sock[i].sin_addr.s_addr = inet_addr (inet_ntoa (in));
	}
      remote_sock[i].sin_port = htons (sock_barr_port);
    }

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Created the socket for the barrier\n");
#endif

}


/* 
   Master node: Recv barr mesg from all other nodes. 
   Send barr mesg to all other nodes
*/
void
MasterSocketBarrier ()
{
  int i;
  int ret;
  int n = strlen (BARR_SOCK_MESG);
  int count = 0;
  static int sock_barr_cnt = 0;
  char recvline[128];

  sock_barr_cnt++;

  /*
    Wait for barrier messages from all the other nodes
  */
  while (count < (HLRC_nprocs - 1))
    {
      if ((n =
	   recvfrom (barr_sock, recvline, strlen (BARR_SOCK_MESG) + 1, 0,
		     (struct sockaddr *) 0, (int *) 0)) < 0)
	{
	  perror ("recvfrom");
	  HLRC_ErrPrint ("Recvfrom Error: Received %d messages\n", count);
	}
      count++;
    }

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Halfway through socket barrier\n");
  fflush (stderr);
#endif

  /*
    Send a barrier message to everyone
  */
  strcpy (sendline, BARR_SOCK_MESG);
  n = strlen (sendline);
  for (i = 1; i < HLRC_nprocs; i++)
    {
#ifdef HLRC_DEBUG
      HLRC_ErrPrint ("Sending barr mesg to %d\n", i);
      fflush (stderr);
#endif
      if ((ret =
	   sendto (barr_sock, sendline, n, 0,
		   (struct sockaddr *) &remote_sock[i],
		   sizeof (remote_sock[i]))) != n)
	{
	  HLRC_ErrPrint ("sendto error on socket: %d\n", ret);
	  perror ("");
	  exit (1);
	}
    }
}

/* 
   Slave nodes: Send barr mesg to master node. 
   Wait for barr mesg from master node
*/
void
SlaveSocketBarrier ()
{
  int i;
  int ret;
  int n = strlen (BARR_SOCK_MESG);
  int count = 0;
  static int sock_barr_cnt = 0;
  char recvline[128];

  sock_barr_cnt++;

  /*
    Send barrier mesg to master
  */
  strcpy (sendline, BARR_SOCK_MESG);
  n = strlen (sendline);
  if ((ret =
       sendto (barr_sock, sendline, n, 0, (struct sockaddr *) &remote_sock[0],
	       sizeof (remote_sock[0]))) != n)
    {
      HLRC_ErrPrint ("sendto error on socket: %d\n", ret);
      perror ("");
      exit (1);
    }

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Halfway through socket barrier\n");
  fflush (stderr);
#endif

  /*
    Wait for barrier mesg from master
  */
  if ((n = recvfrom (barr_sock, recvline, strlen (BARR_SOCK_MESG) + 1, 0,
		     (struct sockaddr *) 0, (int *) 0)) < 0)
    {
      perror ("recvfrom");
      HLRC_ErrPrint ("Recvfrom Error: Received %d messages\n", count);
    }

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Completed socket barrier: %d\n", sock_barr_cnt);
  fflush (stderr);
#endif
}


/*
  Invoke MasterSocketbarrier for node 0 and SlaveSocketBarrier
  for node 1
*/
void
SocketBarrier ()
{
  if (HLRC_myself)
    {
      SlaveSocketBarrier ();
    }
  else
    {
      MasterSocketBarrier ();
    }
}


/*
  Aligned malloc: Allocate memory and align it on the 
  specified boundary
*/
char *
_AMALLOC (int size, int alignment)
{
  char *ptr;
  unsigned mask = alignment - 1;
  ptr = (char *) malloc (size + alignment - 1);

  if (ptr != NULL && ((unsigned) ptr & mask) != 0)
    {
      memset (ptr, 0, size + alignment - 1);
      ptr = (char *) (((unsigned) ptr + mask) & ~mask);
    }

  return ptr;
}


/*
  Allocate page-aligned memory from the pre-allocated 
  shared memory pool. 
*/
void *
HLRC_Malloc (int size)
{
  register int relpage;
  register int i;
  char *addr;
  unsigned int tmp;
  unsigned pageno;
  int npage;
  int error;
  char *c;
  int diff;

  if (size & 3)  {
    size = ((size + WORDSIZE) >> 2) << 2;
  }
  if (diff = (size % pagesize))  {
    size = ((size + pagesize) >> logpagesize) << logpagesize;
  }

  tmp = (int) _first_alloc;

  _first_alloc += (size / WORDSIZE);

  npage = size / pagesize;
  pageno = tmp >> logpagesize;

  HLRC_total_pages += npage;
  HLRC_stat.allocated += npage;

  if (!_first_shared_page) {
    _first_shared_page = pageno;
  }
  else if ((pageno + npage -_first_shared_page) > (PAGE_NUM - _size_adjust)) {
    HLRC_ErrPrint ("Not enough shared memory (size requested: %i)\n", size);
    HLRC_ErrPrint ("Try increasing PAGE_NUM(currently %d, available %d, total %d)\n", PAGE_NUM, (pageno + npage - _first_shared_page), (PAGE_NUM - _size_adjust));
    printf("entrou no if 2\n");
    exit (1);
  }

  for (relpage = pageno - _first_shared_page, i = 0; i < npage; i++)
    {
      if (i < (npage - 1))
	{
	  _per_page[relpage + i].size = pagesize;
	}
      else if (diff)
	{
	  _per_page[relpage + i].size = diff;
	}
      else
	{
	  _per_page[relpage + i].size = pagesize;
	}

      _per_page[relpage + i].home = -1;
    }
  _last_shared_page = pageno + (npage - 1);

  for (c = (char *) tmp, i = 0; i < (size); i++)
    {
      *(c + i) = '\0';
    }

  return (void *) tmp;
}


/*
  Optimize mprotect by accumulating multiple mprotects 
  for contiguous pages, and achieving the protection
  through a single mprotect call for the accumulated
  block of memory. add_mprotect is called with a negative
  value for relpage for this final mprotect.
*/
void 
add_mprotect (int relpage, int protection)
{
  int errc;
  static int begin;
  static int size;

#ifdef HLRC_DEBUG
  fprintf (logfile, "Setting protection for page: %d to %d\n", relpage,
	   protection);
  fflush (logfile);
#endif

  if (relpage < 0)
    {
      if (size)
	{
	  if (errc =
	      mprotect ((void *) ((begin + _first_shared_page) <<
				  logpagesize), size * pagesize, protection))
	    {
	      fprintf (logfile, "add_mprotect: mprotect error: page %d\n",
		       relpage);
	      fflush (logfile);
	      perror ("add_mprotect");
	      exit (1);
	    }
	  size = 0;
	}
      return;
    }
  if ((begin + size) == relpage)
    {
      size++;
    }
  else
    {
      if (errc =
	  mprotect ((void *) ((begin + _first_shared_page) << logpagesize),
		    size * pagesize, protection))
	{
	  fprintf (logfile, "add_mprotect: mprotect error: page %d\n",
		   relpage);
	  fflush (logfile);
	  perror ("add_mprotect");
	  exit (1);
	}
      begin = relpage;
      size = 1;
    }
}


/* -------------------------------------------------------------------*/

/*
  Enter the critical section identified by num.
*/
void
HLRC_EnterCriticalSection (int num)
{
#ifdef HLRC_DEBUG
  fprintf (logfile, "%d Trying to enter critical section(%d)\n",
	   (int) pthread_self(), num);
  fflush (logfile);
#endif

  pthread_mutex_lock (&HLRC_critical_section_mutex[num]);

#ifdef HLRC_DEBUG
  fprintf (logfile, "%d: Entered critical section(%d)\n", (int) 
	   pthread_self(), num);
  fflush (logfile);
#endif
}


/*
  leave the critical section identified by num.
*/
void
HLRC_LeaveCriticalSection (int num)
{
  pthread_mutex_unlock (&HLRC_critical_section_mutex[num]);

#ifdef HLRC_DEBUG
  fprintf (logfile, "%d: Exited critical section(%d)\n", (int)
	   pthread_self(), num);
  fflush (logfile);
#endif
}

void TRANS_init_comm ()
{
  int i;
  struct hostent *phostent;

#ifdef HLRC_TCP
  //fprintf(logfile,"Using TCP protocol\n");
  fflush(logfile);
  //HLRC_ErrPrint("Using TCP protocol\n");
#else
  fprintf(logfile,"Using UDP protocol\n");
  fflush(logfile);
  //HLRC_ErrPrint("Using UDP protocol\n");
#endif

  for (i = 0; i < HLRC_nprocs; i++)
    {
      if (i == HLRC_myself)
	continue;

      if ((phostent =
	   gethostbyname (HLRC_hostnames[i])) == (struct hostent *) NULL)
	{
	  HLRC_ErrPrint ("Host %s not found.\n", HLRC_hostnames[i]);
	  exit (1);
	}
      else
	{
	  struct in_addr in;
	  memcpy (&in.s_addr, phostent->h_addr_list[0], sizeof (in.s_addr));
	  otherInAddr[i].s_addr = in.s_addr;
	}

      // Initialze the sockets for the server thread

#ifdef HLRC_TCP
      if ((accSock[i] = socket (AF_INET, SOCK_STREAM, 0)) < 0){
	HLRC_ErrPrint ("socket(): accSock[%d] failed\n", i);
	exit (1);
      }
#else
      if ((servSock[i] = socket (AF_INET, SOCK_DGRAM, 0)) < 0){
	HLRC_ErrPrint ("socket(): accSock[%d] failed\n", i);
	exit (1);
      }
#endif

#ifdef HLRC_TCP

      if ((rdmaAccSock[i] = socket (AF_INET, SOCK_STREAM, 0)) < 0){
	HLRC_ErrPrint ("socket(): rdmaAccSock[%d] failed\n", i);
	exit(1);
      }
#else
      if ((rdmaServSock[i] = socket (AF_INET, SOCK_DGRAM, 0)) < 0){
	HLRC_ErrPrint ("socket(): rdmaServSock[%d] failed\n", i);
	exit(1);
      }
#endif

      servAddr.sin_port = htons(BASE_SERV_PORT + i);
      rdmaServAddr.sin_port = htons(BASE_RDMA_SERV_PORT + i);

#ifdef HLRC_TCP
      if (bind (accSock[i], (SA *) & servAddr, sizeof (servAddr)) != 0){
	HLRC_ErrPrint ("bind(): accSock[%d] failed\n", i);
	exit(1);
      }
#else
      if (bind (servSock[i], (SA *) & servAddr, sizeof (servAddr)) != 0){
	HLRC_ErrPrint ("bind(): servSock[%d] failed\n", i);
	exit(1);
      }
#endif

#ifdef HLRC_TCP
      if (bind (rdmaAccSock[i], (SA *) & rdmaServAddr, sizeof (rdmaServAddr)) != 0){
	HLRC_ErrPrint ("bind(): rdmaAccSock[%d] failed\n", i);
	exit(1);
      }
#else
      if (bind (rdmaServSock[i], (SA *) & rdmaServAddr, sizeof (rdmaServAddr)) != 0){
	HLRC_ErrPrint ("bind(): rdmaServSock[%d] failed\n", i);
	exit(1);
      }
#endif

      /* Pay attention!!  For TCP is necessary two separeted loops because
	 the blocking accepts */

#ifdef HLRC_TCP

      if (listen(accSock[i],1) != 0){
	HLRC_ErrPrint("listen(): accSock[%d] failed\n",i);
      }

      if (listen(rdmaAccSock[i],1) != 0){
	HLRC_ErrPrint("listen(): rdmaAccSock[%d] failed\n",i);
      }

    }

  SocketBarrier();

  for (i = 0; i < HLRC_nprocs; i++){

    if (i == HLRC_myself)
      continue;

    // Initialize the sockets for send and receive messages between application threads
    if ((appSock[i] = socket (AF_INET, SOCK_STREAM, 0)) < 0){
      HLRC_ErrPrint ("socket(): appSock[%d] failed\n", i);
      exit (1);
    }

    // Initialize the sockets for send and receive rdma messages between application threads
    if ((rdmaAppSock[i] = socket (AF_INET, SOCK_STREAM, 0)) < 0){
      HLRC_ErrPrint ("socket(): rdmaAppSock[%d] failed\n", i);
      exit (1);
    }

#else
    if ((appSock[i] = socket (AF_INET, SOCK_DGRAM, 0)) < 0){
      HLRC_ErrPrint ("socket(): appSock[%d] failed\n", i);
      exit (1);
    }

    if ((rdmaAppSock[i] = socket (AF_INET, SOCK_DGRAM, 0)) < 0){
      HLRC_ErrPrint ("socket(): appSock[%d] failed\n", i);
      exit (1);
    }

#endif

    appAddr.sin_addr = otherInAddr[i];
    rdmaAppAddr.sin_addr = otherInAddr[i];

    if (connect (appSock[i], (SA *) & appAddr, sizeof (appAddr)) < 0){
      HLRC_ErrPrint ("connect(): appSock[%d] failed\n", i);
      exit (1);
    }

    if (connect (rdmaAppSock[i], (SA *) & rdmaAppAddr, sizeof (rdmaAppAddr)) < 0){
      HLRC_ErrPrint ("connect(): rdmaAppSock[%d] failed\n", i);
      exit (1);
    }

#ifdef HLRC_TCP
    if ((servSock[i] = accept(accSock[i], NULL, NULL)) < 0){
      HLRC_ErrPrint("accept(): accSock[%d] failed\n",i);
      exit(1);
    }

    if ((rdmaServSock[i] = accept(rdmaAccSock[i], NULL, NULL)) < 0){
      HLRC_ErrPrint("accept(): rdmaAccSock[%d] failed\n",i);
      exit(1);
    }

#endif

    FD_SET (servSock[i], &req_socks);
    FD_SET (rdmaServSock[i], &rdma_socks);
    max_sock = MAX(max_sock,servSock[i]);
    rdma_max_sock = MAX(rdma_max_sock, rdmaServSock[i]);
    // Put the initilized socket in the fd_set for I/O multiplexing
  }
}

// Initialize the transport protocol

TRANS_RETURN TRANS_Init ()
{
  int i;

  bzero((SA *)&servAddr, sizeof (servAddr));
  bzero((SA *)&appAddr, sizeof (appAddr));
  bzero((SA *)&rdmaAppAddr, sizeof(rdmaAppAddr));
  bzero((SA *)&rdmaServAddr, sizeof(rdmaServAddr));

  // the variable is the IP address
  appAddr.sin_family = AF_INET;
  rdmaAppAddr.sin_family = AF_INET;

  appAddr.sin_port = htons (BASE_SERV_PORT + HLRC_myself);
  rdmaAppAddr.sin_port = htons(BASE_RDMA_SERV_PORT + HLRC_myself);

  // the variable is the PORT
  servAddr.sin_family = rdmaServAddr.sin_family = AF_INET;
  servAddr.sin_addr.s_addr = rdmaServAddr.sin_addr.s_addr = htonl (INADDR_ANY);

  FD_ZERO (&req_socks);
  FD_ZERO (&rdma_socks);
  TRANS_init_comm ();
  return 0;
}

/*
  Allocate memory for all the buffers to
  be used for RDMA operations in the protocol.
*/

TRANS_RETURN TRANS_RegisterGeneralBuffers (int nodeId)
{
  // nesta rotina foi posto o codigo de registro de buffers utilizados pelo VIa
  // que por motivos de compatibilidade tambem foram postos aqui  
  int i, j;
  int bufId;

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("RegisterGeneralBuffers: %d\n", nodeId);
#endif

  if (nodeId != HLRC_myself)
    {

#ifdef COPY_ON_RECV
      pPageBuffs[nodeId] =
	(TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
				    DESCRIPTOR_ALIGNMENT);
#endif

      pProtocolControls[nodeId] =
	(TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
				    DESCRIPTOR_ALIGNMENT);

      pBarrierBuffs[nodeId] =
	(TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
				    DESCRIPTOR_ALIGNMENT);

      for (bufId = 0; bufId < MAX_SIMUL_MESG; bufId++)
	{
	  pRecvBuffs[nodeId][bufId] =
	    (int *) _AMALLOC (2 * sizeof (int), DESCRIPTOR_ALIGNMENT);
	}

      for (j = 0; j < HLRC_nprocs; j++)
	{
	  pRequestBuffs[nodeId][j] =
	    (TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
					DESCRIPTOR_ALIGNMENT);

	  pRemoteBins[nodeId][j] =
	    (TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
					DESCRIPTOR_ALIGNMENT);
	}

      pBarrEndBuffs[nodeId] =
	(TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
				    DESCRIPTOR_ALIGNMENT);
    }

  return TRANS_SUCCESS;
}

/*
  Complete the receive of DMA mappings in the buffers to
  be used for RDMA operations in the protocol.
*/

TRANS_RETURN TRANS_ReceiveDMAMappings (int source, int index)
{
  int count = 0;
  int i;
  TRANS_msg_mapp msg;

  if (TRANS_recv_rdma_mapp (&msg,servSock[source]) < 0)
    if (errno != EINTR)
      {
	HLRC_ErrPrint ("Error receiving DMA map from: %d\n",source);
	return errno;
      }

  switch (msg.type)
    {
    case START_ADDR_MESG:
      *pRemoteStartAddr[source] = (int) msg.addr;
      break;
#ifdef PROT_HANDLE
    case SVM_ADDR_MESG:
      pRemoteMemory[source][index]->pAddress = msg.addr;
      break;
#endif
    case DIFFACK_MESG:
      pProtocolControls[source]->pAddress = msg.addr;
      break;
    case BARRIER_MESG:
      pBarrierBuffs[source]->pAddress = msg.addr;
      break;
    case REQUEST_MESG:
      pRequestBuffs[source][index]->pAddress = msg.addr;
      break;
    case BIN_MESG:
      pRemoteBins[source][index]->pAddress = msg.addr;
      break;
    case DIFF_MESG:
      pDiffBuffs[source]->pAddress = msg.addr;
      break;
    case BARR_END_MESG:
      pBarrEndBuffs[source]->pAddress = msg.addr;
      break;
    case PAGE_MESG:
#ifdef COPY_ON_RECV
      pPageBuffs[source]->pAddress = (char *) msg.addr;
#else
      pRemoteMemory[source][index]->pAddress = (char *) msg.addr;
#endif
      break;
    default:
      HLRC_ErrPrint ("Unknown RDMA type: %d, from: %d\n", msg.type,source);
      break;
    };
  return TRANS_SUCCESS;
}

/*
  Set up the descriptors and buffers according to the type
  of the message and send the DMA mappings.
*/

TRANS_RETURN TRANS_SendDMAMappings (int buff_type, int nodeId, int index)
{
  int size;
  char *part_start_addr;
  void *pAddress = NULL;
  TRANS_msg_mapp msg;

  if (buff_type == START_ADDR_MESG)
    {
      size = sizeof (int);
    }
  else
    {
      size = sizeof (*pAddress);
    }

  /*
    Set up the send buffers and handles
  */

#ifdef PROT_HANDLE
  if (buff_type == SVM_ADDR_MESG)
    {
      part_start_addr =
	(char *) prot_first_alloc + (index * MAX_REGMEM_PAGES * pagesize);

      fprintf (logfile, "part_start_addr[%d]: %x\n", index, (unsigned int)part_start_addr);
      fflush (logfile);

      pAddress = (char *) part_start_addr;
    }
#endif

#ifdef COPY_ON_RECV
  if (buff_type == PAGE_MESG)
    {
      pAddress = (char *) HLRC_page_recv_buff;
    }
#endif
  switch (buff_type){
  case DIFFACK_MESG:
    pAddress = (char *) hlrc_protocol_control;
    break;
  case BARRIER_MESG:
    pAddress = (char *) _bar_buff;
    break;
  case REQUEST_MESG:
    pAddress = (char *) HLRC_request_buffer[index];
    break;
  case BIN_MESG:
    pAddress = (char *) _bin[index];
    break;
  case DIFF_MESG:
    pAddress = (char *) _diff_buff_recv[nodeId];
    break;
  case BARR_END_MESG:
    pAddress = (char *) _bar_end_buff;
    break;
  }

  /*
    Set up the descriptors for the send
  */

  if (nodeId != HLRC_myself)
    {
      msg.size = size;
      msg.type = buff_type;
      if (buff_type == START_ADDR_MESG)
	msg.addr = (char *) *start_addr;
      else
	msg.addr = (char *) pAddress;
      if (TRANS_send_rdma_mapp (&msg, appSock[nodeId]) != TRANS_SUCCESS)
	{  
	  HLRC_ErrPrint ("Unable to send RDMA initialization data\n");
	  return TRANS_ERROR;
	}
      //      HLRC_ErrPrint("Saiu aqui\n");
      return TRANS_SUCCESS;
    }
  return TRANS_ERROR;
}


TRANS_RETURN TRANS_BlockForReceive (recv_t *ret, recv_t *ctrl_ret){
  int i;
  int sock = -1;
  int immData = -1;
  TRANS_msg msg;
  TRANS_msg msg_ctrl;
  fd_set read_fds = req_socks;
  static unsigned long long time_handler_begin = 0, time_handler_end = 0;

  GETTIME (time_handler_end);
  if (time_handler_begin != 0)
    {
      if (!HLRC_clock_stopped)
	{
	  HLRC_stat.time_handler += (time_handler_end - time_handler_begin);
	}
    }

  while (select(max_sock + 1,(fd_set *)&read_fds,NULL,NULL,NULL) < 0)
    if (errno != EINTR)
      {
	HLRC_ErrPrint ("Error: select() of BlockForReceive\n");
	return errno;
      }

  for (i = 0; i < HLRC_nprocs; i++)
    {
      sock = servSock[i];
      if (FD_ISSET (sock, &read_fds)){
	break;
      }
    }


  GETTIME (time_handler_begin);

  HLRC_EnterCriticalSection (0);

#ifdef HLRC_DEBUG
  fprintf (logfile, "Received immediate data: %x from: %d\n", immData, i);
  fflush (logfile);
#endif

  if (TRANS_wait_req (&msg, &msg_ctrl, sock) < 0)
    {
      HLRC_ErrPrint ("Error receiving a message from %d\n", i);
      return errno;
    }

  immData = msg.header;

  ret->mesgType = MESG_TYPE (immData);
  ret->senderId = SENDER_ID (immData);
  ret->bufId = BUFFER_ID (immData);
  ret->nodeId = NODE_ID (immData);

  immData = msg_ctrl.header;

  ctrl_ret->mesgType = MESG_TYPE (immData);
  ctrl_ret->senderId = SENDER_ID (immData);
  ctrl_ret->bufId = BUFFER_ID (immData);
  ctrl_ret->nodeId = NODE_ID (immData);

  return TRANS_SUCCESS;
}

/*
  Export the local buffer to the outside world and obtain 
  handles to the buffers on remote nodes by importing them
*/


TRANS_RETURN TRANS_ExchangeDMAMappings (int buff_type, int index)
{
  int nodeId;
  for (nodeId = 0; nodeId < HLRC_nprocs; nodeId++){
    if (nodeId != HLRC_myself){
      if (TRANS_SendDMAMappings (buff_type, nodeId, index) != TRANS_SUCCESS){
	HLRC_ErrPrint ("Unable to send buff mappings(%d)\n", buff_type);
	return TRANS_ERROR;
      }
    }
  }

  for (nodeId = 0; nodeId < HLRC_nprocs; nodeId++)
    if (nodeId != HLRC_myself)
      if (TRANS_ReceiveDMAMappings (nodeId, index) != TRANS_SUCCESS){
	HLRC_ErrPrint ("Unable to receive buff mappings(%d)\n", buff_type);
	return TRANS_ERROR;
      }

  return TRANS_SUCCESS;
};

/*
  HLRC_SendMessage:
  Send an RDMA write message to the destination node. flag 
  indicates whether the routine was invoked from the application
  thread or server thread. Separate send descriptors are used 
  for the server thread and the application thread. Checks first 
  to see if the send descriptor has been freed after the 
  previous send. If not, waits until the descriptor becomes 
  free. Then, sets up the data for the send and posts the send. 
  There is no receive operation on the destination node for 
  these messages. 
*/

void
HLRC_SendMessage (int dest, int source, int offset,
		  char *Data, int size, int type, int num_part)
{
  int index;
  int sock;
  int ret;
  int bufId;
  int i;
  char *ptr;
  TRANS_msg msg;
  sock = appSock[dest];

#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] Sending %d to node:%d[%d](offset:%d, size:%d), %p %i\n",
	   (int)pthread_self (), type, dest, source, offset, size, Data,
	   num_part);
  fflush (logfile);
#endif

  if (type == DIFF && offset >= sizeof (TDiffControl))
    bufId = 1;
  else
    bufId = 0;
#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] 1\n",
	   (int)pthread_self ());
  fflush (logfile);
#endif

  msg.header = MESG_HDR (type, HLRC_myself, bufId, source);
  msg.local.length = size;
  msg.local.addr = Data;

#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] 2\n",
	   (int)pthread_self ());
  fflush (logfile);
#endif


  switch (type)
    {

#ifndef DIRECT_DIFFS
    case DIFF_MESG:
      msg.remote.addr =
	(char *) ((int) (pDiffBuffs[dest]->pAddress) + offset);
      break;
#endif

    case DIFFACK_MESG:
      msg.remote.addr =
	(char *) ((int) pProtocolControls[dest]->pAddress + offset);
      break;

    case PAGE_MESG:

#ifdef COPY_ON_RECV
      msg.remote.addr = (char *) ((int) pPageBuffs[dest]->pAddress + offset);
#else
      msg.remote.addr =
	(char *) ((int) pRemoteMemory[dest][num_part]->pAddress + offset);
#endif
      break;

    case REQUEST_MESG:
      msg.remote.addr =
	(char *) ((int) pRequestBuffs[dest][source]->pAddress + offset);
      break;

    case BIN_MESG:
      msg.remote.addr =
	(char *) ((int) pRemoteBins[dest][source]->pAddress + offset);

#ifdef HLRC_DEBUG

      fprintf (logfile, "[%d] 3 %p\n",
	       (int)pthread_self (), pRemoteBins[dest][source]->pAddress);
      fflush (logfile);
#endif

      /*      if (pRemoteBins[dest][source]->pAddress == 0) {
	      fprintf(logfile,"pRemoteBins[%d][%d]: %x\n",dest,source,
	      pRemoteBins[dest][source]->pAddress);fflush(logfile);
	      }*/
      break;

    case BARRIER_MESG:
      msg.remote.addr =
	(char *) ((int) pBarrierBuffs[dest]->pAddress + offset);
      break;

    case BARR_END_MESG:
      msg.remote.addr =
	(char *) ((int) pBarrEndBuffs[dest]->pAddress + offset);
      break;
    case END_MESG:
      msg.remote.addr = NULL;
      break;
    default:
      HLRC_ErrPrint ("Invalid message passed to SendMessage\n");
      fprintf (logfile, "Invalid message passed to SendMessage\n");
      fflush (logfile);
      exit (1);
    }
#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] 4\n",
	   (int)pthread_self ());
  fflush (logfile);
#endif

  if ((ret = TRANS_send_req (&msg, sock)) < 0)
    {
      HLRC_ErrPrint ("Error(%d) in posting send in SendMessage\n", ret);
      HLRC_ErrPrint ("Unable to send request\n");
      exit (1);
    }
}

/*
  HLRC_SendMessageCtrl

  This routine makes the same as HLRC_SendMessage, but, it send the control
  message together the data message.
  In the original HLRC it is done by another message sent with SendRequest.
  This modification was done in order to reduce the number of messages.  In UDP/TCP
  this aproach is attractive because the overhead of send a messge.
  This routine is used only to send locks, diffs and pages.
*/

void
HLRC_SendMessageCtrl (int dest, int source, int offset,
		      char *Data, int size, int type, int num_part,
		      int offset_ctrl, char *Data_ctrl, int size_ctrl, int type_ctrl){
  int index;
  int sock;
  int ret;
  int bufId;
  int i;
  char *ptr;
  TRANS_msg msg,msg_ctrl;

  sock = appSock[dest];

  if (type == DIFF && offset >= sizeof (TDiffControl))
    bufId = 1;
  else
    bufId = 0;

  msg.header = MESG_HDR (type, HLRC_myself, bufId, source);
  msg.local.length = size;
  msg.local.addr = Data;

  if (type_ctrl == DIFF && offset >= sizeof(TDiffControl))
    bufId = 1;
  else
    bufId = 0;

  msg_ctrl.header = MESG_HDR(type_ctrl, HLRC_myself, bufId, source);
  msg_ctrl.local.length = size_ctrl;
  msg_ctrl.local.addr = Data_ctrl;
  msg_ctrl.remote.addr = (char *) ((int) pRequestBuffs[dest][source]->pAddress + offset);

  switch (type){
#ifndef DIRECT_DIFFS
  case DIFF_MESG:
    msg.remote.addr =
      (char *) ((int) (pDiffBuffs[dest]->pAddress) + offset);
    break;
#endif
  case PAGE_MESG:
#ifdef COPY_ON_RECV
    msg.remote.addr = (char *) ((int) pPageBuffs[dest]->pAddress + offset);
#else
    msg.remote.addr =
      (char *) ((int) pRemoteMemory[dest][num_part]->pAddress + offset);
#endif
    break;

  case BIN_MESG:
    msg.remote.addr =
      (char *) ((int) pRemoteBins[dest][source]->pAddress + offset);
    break;

    // analizar mensagens abaixo

  case BARRIER_MESG:
    msg.remote.addr =
      (char *) ((int) pBarrierBuffs[dest]->pAddress + offset);
    break;

  case BARR_END_MESG:
    msg.remote.addr =
      (char *) ((int) pBarrEndBuffs[dest]->pAddress + offset);
    break;
  default:
    HLRC_ErrPrint ("Invalid message passed to SendMessage\n");
    fprintf (logfile, "Invalid message passed to SendMessage\n");
    fflush (logfile);
    exit (1);
  }

  if ((ret = TRANS_send_req_ctrl (&msg, &msg_ctrl, sock)) < 0){
    HLRC_ErrPrint ("Error(%d) in posting send in SendMessage\n", ret);
    HLRC_ErrPrint ("Unable to send message\n");
    exit (1);
  }
}

#ifdef DIRECT_DIFFS
void
HLRC_SendDiff (int flag, int dest, int source, int offset,
	       char *Data, int size, int type, int num_part)
{
  int index;

  TRANS_msg msg;

#ifdef HLRC_DEBUG

  fprintf (logfile,
	   "[%d] Sending DIFF to node:%d[%d](offset:%d, size:%d), %p\n",
	   (int)pthread_self (), dest, source, offset, size, Data);
  fflush (logfile);
#endif

  msg.header = MESG_HDR(type,HLRC_myself,1,source);
  msg.local.length = size;
  msg.local.addr = Data;

#ifdef HLRC_DEBUG
  fprintf (logfile, "Now we have a descriptor for send\n");
  fflush (logfile);
#endif

  switch (type)
    {
    case DIFF_MESG:
      msg.remote.addr = (char *)((int)pRemoteMemory[dest][num_part]->pAddress + offset);
      break;
    default:
      HLRC_ErrPrint ("Invalid message passed to SendDiff\n");
      fprintf (logfile, "Invalid message passed to SendDiff\n");
      fflush (logfile);
      exit (1);
    }

  if (TRANS_send_req(&msg,appSock[dest]) != TRANS_SUCCESS){
    HLRC_ErrPrint("Error sending DIFF message\n");
    fprintf(logfile,"Error sending DIFF message\n");
    exit(1);
  }
}

#endif

/*
  HLRC_FreeTwin:
  Free memory used for twin. Don't release the memory to
  the system - keep it around for possible reuse later.
*/
void
HLRC_FreeTwin (int relpage)
{
  _free_twins[_notwins++] = _per_page[relpage].twin;
  _per_page[relpage].twin = 0;

  return;
}

#ifdef HLRC_TCP

TRANS_RETURN TRANS_recv(int from, char *msg, int size){
  int recebidos = 0;
  int n, r = size;
  while (recebidos < size){
    n = read(from,msg,r);
    if (n < 0){
      if (errno != EINTR)
	return TRANS_ERROR;
    }else if (n > 0){
      recebidos += n;
      msg += n;
      r -= n;
    }
  }
  return recebidos;
}

#else

TRANS_RETURN TRANS_recv(int from, char *msg, int size){
  int recebidos = 0;
  int n;
  int variavel;
  char *ptr;
  int pendente = 1;
  ptr = msg;
  while (recebidos < size){
    n = read(from,ptr,size);
    recebidos += n;
    if ((n < 0)&&(errno != EINTR)){
      return TRANS_ERROR;
    } 
    if ((recebidos >= 8)&&(pendente)){
      memcpy(&variavel, msg + 4, 4);
      size = variavel + 12;
      pendente = 0;
    }
    ptr += n;
  }
  return n;
}

#endif

TRANS_RETURN TRANS_send(int to, char *msg, int size){
  int enviados = 0;
  int n, r = size;
  while (enviados < size){
    n = write(to, msg,r);
    if (n < 0){
      if (errno != EINTR)
	return TRANS_ERROR;
    }else if (n > 0) {
      enviados += n;
      r -= n;
      msg += n;
    }
  }
  return enviados;
}

TRANS_RETURN TRANS_wait_req (TRANS_msg * msg, TRANS_msg * msg_ctrl, int from){
  int n;
  int size;
  char *ptr;

#ifdef HLRC_TCP
  size = sizeof(msg->header) + sizeof(msg->local.length) + sizeof(msg->local.addr);
  n = TRANS_recv(from, buffer_pkg_recv, size);
#else
  n = TRANS_recv(from, buffer_pkg_recv, BUFFER_PKG_SIZE);
#endif

  if (n < 0)
    goto cleanup_error;

  ptr = buffer_pkg_recv;

  memcpy(&msg->header, ptr, sizeof(msg->header));

  ptr += sizeof(msg->header);

  memcpy(&msg->local.length, ptr, sizeof(msg->local.length));
  ptr += sizeof(msg->local.length);

  memcpy(&msg->local.addr, ptr, sizeof(msg->local.addr));

  ptr += sizeof(msg->local.addr);

  if (msg->local.length < 0){  // IT'S NECESSARY COPY THE CONTROL DATA
#ifdef HLRC_TCP
    n = TRANS_recv(from, ptr, size);
#endif
    msg->local.length = -1 * msg->local.length;

    memcpy(&msg_ctrl->header, ptr, sizeof(msg_ctrl->header));    
    ptr += sizeof(msg_ctrl->header);

    memcpy(&msg_ctrl->local.length, ptr, sizeof(msg_ctrl->local.length));
    ptr += sizeof(msg_ctrl->local.length);

    memcpy(&msg_ctrl->local.addr, ptr, sizeof(msg_ctrl->local.addr));
    ptr += sizeof(msg_ctrl->local.addr);

#ifdef HLRC_TCP
    n = TRANS_recv(from, msg_ctrl->local.addr, msg_ctrl->local.length);
#else  
    memcpy(msg_ctrl->local.addr, ptr, msg_ctrl->local.length);
#endif

    ptr += msg_ctrl->local.length;
  }

#ifdef HLRC_TCP
  n = TRANS_recv(from, msg->local.addr, msg->local.length);
#else
  memcpy(msg->local.addr, ptr, msg->local.length);
#endif

  if (n < 0)
    goto cleanup_error;

 cleanup_error:
  if (n == TRANS_ERROR)
    return TRANS_ERROR;
  else return TRANS_SUCCESS;
}

TRANS_RETURN TRANS_send_req (TRANS_msg * msg, int to){
  int size;
  char *ptr;
#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] 5\n",
	   (int)pthread_self ());
  fflush (logfile);
#endif

  sem_wait(&mutex_comm);

#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] 6\n",
	   (int)pthread_self ());
  fflush (logfile);
#endif


  size = sizeof(msg->header) + sizeof(msg->local.length) +
    sizeof(msg->remote.addr) + msg->local.length;

  ptr = buffer_pkg_send;

  memcpy(ptr, &msg->header, sizeof(msg->header));
  ptr += sizeof(msg->header);

  memcpy(ptr, &msg->local.length, sizeof(msg->local.length));
  ptr += sizeof(msg->local.length);

  memcpy(ptr, &msg->remote.addr, sizeof(msg->remote.addr));
  ptr += sizeof(msg->remote.addr);

#ifdef HLRC_DEBUG

  fprintf (logfile, "[%d] 7 %p %p %d\n",
	   (int)pthread_self (), ptr, msg->local.addr, msg->local.length);
  fflush (logfile);
#endif


  memcpy(ptr, msg->local.addr, msg->local.length);

#ifdef HLRC_DEBUG
  fprintf (logfile,
	   "[%d] Sending %p to node: %d\n",
	   (int)pthread_self (), msg, to);
  fflush (logfile);
#endif


  size = TRANS_send(to, buffer_pkg_send, size);



  sem_post(&mutex_comm);

  if (size == TRANS_ERROR)
    return TRANS_ERROR;
  else return TRANS_SUCCESS;
}

/*
  TRANS_RETURN TRANS_wait_req (TRANS_msg * msg, int from){
  int n;
  char *ptr;
  int fixed_part;
  TRANS_msg msg_ctrl;

  fixed_part = sizeof(msg->header) + sizeof(msg->local.length)
  + sizeof(msg->local.addr);

  #ifdef HLTC_TCP
  n = TRANS_recv(from, buffer_pkg, fixed_part);
  // pegar so a parte fixa senao vai grudar mensagens
  #else
  n = TRANS_recv(from, buffer_pkg, BUFFER_PKG_SIZE);
  #endif

  if (n < 0)
  goto cleanup_error;

  memcpy(&msg->header, buffer_pkg, sizeof(msg->header));

  ptr = buffer_pkg + sizeof(msg->header);

  memcpy(&msg->local.length, ptr, sizeof(msg->local.length));

  ptr += sizeof(msg->local.length);

  memcpy(&msg->local.addr, ptr, sizeof(msg->local.addr));

  ptr += sizeof(msg->local.addr);

  if (msg->local.length < 0){  // IT'S NECESSARY COPY THE CONTROL DATA

  msg->local.length = -1 * msg->local.length;

  memcpy(&msg_ctrl.header, ptr, sizeof(msg_ctrl.header));    
  ptr += sizeof(msg_ctrl.header);

  memcpy(&msg_ctrl.local.length, ptr, sizeof(msg_ctrl.local.length));
  ptr += sizeof(msg_ctrl.local.length);

  memcpy(&msg_ctrl.local.addr, ptr, sizeof(msg_ctrl.local.addr));
  ptr += sizeof(msg_ctrl.local.addr);

  memcpy(msg_ctrl.local.addr, ptr, msg_ctrl.local.length);
  ptr += msg_ctrl.local.length;
  }

  #ifdef HLRC_TCP
  n = TRANS_recv(from, msg->local.addr, msg->local.length);
  #else
  memcpy(msg->local.addr, ptr, msg->local.length);
  #endif

  cleanup_error:

  if (n == TRANS_ERROR)
  return TRANS_ERROR;
  else return TRANS_SUCCESS;
  }
*/

TRANS_RETURN TRANS_send_req_ctrl (TRANS_msg * msg, TRANS_msg * msg_ctrl, int to){
  int size = 2 * (sizeof(msg->header) + sizeof(msg->local.length)
		  + sizeof(msg->local.addr)) + msg->local.length + msg_ctrl->local.length;
  char *pkg,*ptr;
  int len;

  pkg = malloc(size);

  // FIXED PART OF DATA

  memcpy(pkg, &msg->header, sizeof(msg->header));
  ptr = pkg + sizeof(msg->header);

  len = -1 * msg->local.length;
  memcpy(ptr, &len, sizeof(msg->local.length));
  ptr += sizeof(msg->local.length);

  memcpy(ptr, &msg->remote.addr, sizeof(msg->remote.addr));
  ptr += sizeof(msg->remote.addr);

  // FIXED PART OF CONTROL

  memcpy(ptr, &msg_ctrl->header, sizeof(msg_ctrl->header));
  ptr += sizeof(msg_ctrl->header);

  memcpy(ptr, &msg_ctrl->local.length, sizeof(msg_ctrl->local.length));
  ptr += sizeof(msg_ctrl->local.length);

  memcpy(ptr, &msg_ctrl->remote.addr, sizeof(msg_ctrl->remote.addr));
  ptr += sizeof(msg_ctrl->remote.addr);

  // VARIABLE PART OF CONTROL

  memcpy(ptr, &msg_ctrl->local.addr, msg_ctrl->local.length);
  ptr += msg->local.length;

  // VARIABLE PART OF DATA

  memcpy(ptr, msg->local.addr, msg->local.length);

  sem_wait(&mutex_comm);
  size = TRANS_send(to, pkg, size);
  sem_post(&mutex_comm);

  if (size == TRANS_ERROR)
    return TRANS_ERROR;
  else return TRANS_SUCCESS;
}

TRANS_RETURN TRANS_send_rdma_mapp (TRANS_msg_mapp * msg, int to)
{
  int ret;
  sem_wait(&mutex_comm);
  ret = TRANS_send (to, (char *) &msg->type, sizeof(msg->type));
  if (ret == TRANS_ERROR)
    goto rdma_clean_error;
  ret = TRANS_send (to, (char *) &msg->size, sizeof(msg->size));
  if (ret == TRANS_ERROR)
    goto rdma_clean_error;
  ret = TRANS_send (to, (char *) &msg->addr, sizeof(msg->addr));
  if (ret == TRANS_ERROR)
    goto rdma_clean_error;
  sem_post(&mutex_comm);
  return TRANS_SUCCESS;

 rdma_clean_error:
  sem_post(&mutex_comm);
  return TRANS_ERROR;
}

TRANS_RETURN TRANS_recv_rdma_mapp (TRANS_msg_mapp * msg, int from)
{
  int ret;
  ret = TRANS_recv (from,(char *)&msg->type, sizeof(msg->type));
  if (ret == TRANS_ERROR)
    return TRANS_ERROR;
  ret = TRANS_recv (from, (char *)&msg->size, sizeof(msg->size));
  if (ret == TRANS_ERROR)
    return TRANS_ERROR;
  ret = TRANS_recv (from, (char *)&msg->addr, sizeof(msg->addr));
  if (ret == TRANS_ERROR)
    return TRANS_ERROR;
  return TRANS_SUCCESS;
}

/*
  HLRC_SendRequest:
  Send an RDMA write message to the destination node. flag
  indicates whether the routine was invoked from the application
  thread or server thread. Separate send descriptors are used
  for the server thread and the application thread. Checks first
  to see if the send descriptor has been freed after the
  previous send. If not, waits until the descriptor becomes
  free. Then, sets up the data for the send and posts the send.
  RDMA write is used with immediate data which makes it
  mandatory for receive descriptors to be posted on the 
  destination node. Used to send PAGE/LOCK/DIFF request
  messages.

  A page sized buffer is used for the request. However, only
  a small portion is used for the request. For the PAGE/LOCK
  requests, the following structure is used.

  --------------------------------------------------------------
  Field        Index Variable        Actual Index
  --------------------------------------------------------------
  mesg type      msg_type_index      HLRC_MSG_HDR_TOP
  mesg origin    msg_node_index      HLRC_MSG_HDR_TOP-1 
  item id        msg_item_index      HLRC_MSG_HDR_TOP-2
  (page/lock no)

  time stamp     msg_tmst1_start     HLRC_MSG_HDR_TOP-2-NPROCS
  --------------------------------------------------------------
  Only (NPROCS + MSG_HDR_LEN)*sizeof(int) bytes are used for the request 
  message.

  For the DIFF message, the TDiffControl data structure is used 
  and the top three fields are the same as for a PAGE/LOCK req.
  Only sizeof(TDiffControl) bytes are used for the DIFF message.
*/
void
HLRC_SendRequest (int dest, int source, int offset,
		  volatile int *Data, int size, int type)
{
  int index;			//lobosco
  int immData;
  int bufId;
  int sock;
  int ret;
  int i;
  char *ptr;
  TRANS_msg msg;

  if (dest != HLRC_myself){

#ifdef HLRC_DEBUG
    fprintf (logfile,
	     "[%d] Sending req: %d to node: %d(offset: %d, size: %d)\n",
	     (int)pthread_self (), type, dest, offset, size);
    fflush (logfile);
#endif

    sock = appSock[dest];

    if (type == DIFF && offset >= sizeof (TDiffControl))
      bufId = 1;
    else
      bufId = 0;

    msg.header = MESG_HDR (type, HLRC_myself, bufId, source);
    msg.local.addr = (char *) Data;
    msg.local.length = size;
    msg.remote.addr =	(char *) ((int) pRequestBuffs[dest][source]->pAddress + offset);
    msg.remote.length = 0;
    if ((ret = TRANS_send_req (&msg, sock)) < 0)
      {
	HLRC_ErrPrint ("Error(%d) in posting send in SendRequest\n", errno);
	HLRC_ErrPrint ("Unable to send request\n");
	exit (1);
      }

    return;
  }

  exit (1);
}

/*
  HLRC_Recv:
  Wait on a semaphore for a response to an earlier request
  message and process the received response data.
*/

void
HLRC_Recv (int type, volatile int *addr, int size, int relpage)
{
  register int i;
  int start_page;
  TRANS_msg msg;

#ifdef HLRC_DEBUG
  fprintf (logfile, "Waiting for %d message, size: %d\n", type, size);
  fflush (logfile);
#endif

  switch (type)
    {
    case PAGE:
      sem_wait (&sem_page);
      break;
    case LOCK:
      do {
	sem_wait (&sem_lock);
	if (relpage != HLRC_request_buffer[HLRC_myself][msg_item_index])
	  sem_post(&sem_lock);
      } while (relpage != HLRC_request_buffer[HLRC_myself][msg_item_index]);
      break;
    };

  if (HLRC_request_buffer[HLRC_myself][msg_type_index] != type)
    {
      fprintf (logfile, "Error: Received %d, Expected %d\n",
	       HLRC_request_buffer[HLRC_myself][msg_type_index], type);
      fflush (logfile);
      HLRC_ErrPrint ("Error: Received %d, Expected %d\n",
		     HLRC_request_buffer[HLRC_myself][msg_type_index], type);
      exit (1);
    }

#ifdef HLRC_DEBUG
  fprintf (logfile, "Got %d message for %d from %d\n", type, HLRC_request_buffer[HLRC_myself][msg_item_index], HLRC_request_buffer[HLRC_myself][msg_node_index]);
  fflush (logfile);
#endif

  switch (type)
    {

    case PAGE: 

      if (size > NPROCS)
	{
	  *addr++ = HLRC_request_buffer[HLRC_myself][msg_node_index];
	  if (HLRC_request_buffer[HLRC_myself][msg_node_index] == HLRC_myself)
	    {
	      HLRC_stat.homes++;
	    }
	  size--;
	}

      if (relpage != HLRC_request_buffer[HLRC_myself][msg_item_index])
	{
	  fprintf (logfile, "Error: Received page: %d Expected page: %d\n",
		   HLRC_request_buffer[HLRC_myself][msg_item_index], relpage);
	  fflush (logfile);
	  HLRC_ErrPrint ("Error: Received page: %d Expected page: %d\n",
			 HLRC_request_buffer[HLRC_myself][msg_item_index],
			 relpage);
	  exit (1);
	}

#ifdef COPY_ON_RECV

#ifdef PROT_HANDLE
      start_page = prot_first_shared_page;
#else
      start_page = _first_shared_page;
#endif

#ifdef HLRC_DEBUG
      fprintf (logfile, "Copying onto %x(%d bytes)\n",
	       (relpage + start_page) << logpagesize,
	       _per_page[relpage].size);
      fflush (logfile);
#endif

#ifdef INDIRECT_DIFFS
      if (_per_page[relpage].dirty_and_invalid)
	{
	  int firstsize, tmp_size;
	  int tmp_diff_buff[2048];	/* initialize to 0? */

#ifdef HLRC_DEBUG
	  fprintf (logfile, "Dirty and Invalid\n");
	  fflush (logfile);
#endif

	  firstsize = _compute_diff (_per_page[relpage].twin,
				     HLRC_page_recv_buff, tmp_diff_buff,
				     _per_page[relpage].size / WORDSIZE,
				     &tmp_size);
	  _apply_diff (tmp_diff_buff,
		       (int *) ((relpage + start_page) << logpagesize),
		       firstsize, tmp_size);
	  bcopy (HLRC_page_recv_buff, _per_page[relpage].twin,
		 _per_page[relpage].size);

	  fprintf (logfile, "Dirty and Invalid: Done \n");
	  fflush (logfile);

	  _per_page[relpage].dirty_and_invalid = 0;
	}
      else
	{
	  bcopy (HLRC_page_recv_buff, (relpage + start_page) << logpagesize,
		 _per_page[relpage].size);
	}

#else
      bcopy (HLRC_page_recv_buff, (relpage + start_page) << logpagesize,
	     _per_page[relpage].size);
#endif

#endif /* COPY_ON_RECV */
      bcopy ((const void *)&HLRC_request_buffer[HLRC_myself][msg_tmst1_start], 
	     (void *)addr, size * WORDSIZE);
      break;

    case LOCK:

      for (i = 0; i < HLRC_nprocs; i++)
	{
	  if (i != HLRC_myself)
	    {
	      _apply_wn (HLRC_request_buffer[HLRC_myself]
			 [msg_tmst2_start + i],
			 HLRC_request_buffer[HLRC_myself][msg_tmst1_start +
							  i], i);
	    }
	}

      break;
    }

  HLRC_request_buffer[HLRC_myself][msg_type_index] = 0;
}

/*
  HLRC_ReleaseResources:
  Release the resources used for communication
*/
void
HLRC_ReleaseResources ()
{
  int j;

  SocketBarrier ();

  if (HLRC_myself == 0)
    for (j = 0; j < HLRC_nprocs; j++)
      if (j != HLRC_myself)
	HLRC_SendMessage (j, 0, 0, NULL, 0, END_MESG, 0);

  SocketBarrier();

  close (barr_sock);

  for (j = 0; j < HLRC_nprocs; j++){
    close (appSock[j]);
    close (servSock[j]);
#ifdef HLRC_TCP
    close (rdmaAccSock[j]);
    close (accSock[j]);
#else
    close(rdmaServSock[j]);
#endif
  }
}


/*
  _end_interval: 
  An interval is closed either when a lock is released, or at 
  a barrier. Also, when a lock is acquired to start the 
  interval afresh, the previous interval is closed. For all 
  pages in the dirty list which have some other node as home, 
  send DIFFs to update the copy at home(page).
*/

void
_end_interval ()
{
  register int i;
  register int relpage;
  register int previous = -1;
  int size, home, offset;
  unsigned long long time1, time2;
  int counter = 1;

  HLRC_EnterCriticalSection (1);

  /* 
     Wait for previous barrier to complete. This is important
     because, otherwise, the bin(which contains the write
     notices) can be overwritten by the previous barrier after
     the bin is updated locally.
  */
  if (prev_barr_complete == 0)
    {
      while (counter != HLRC_nprocs)
	{
	  sem_wait(&sem_barr_end);
	  sem_wait(&mutex_barr_end);
	  for (i = 0; i < HLRC_nprocs; i++)
	    {
	      if ((i != HLRC_myself) && (_bar_end_buff[i] >= 0))
		{
		  counter++;
		  _bar_end_buff[i] = -1;
		}
	    }
	  sem_post(&mutex_barr_end);
	}
    };

  prev_barr_complete = 1;

#ifdef HLRC_DEBUG
  fprintf (logfile, "end_interval begin: %d index update: %d\n", _global_stamp[HLRC_myself], _index_update);
  fflush (logfile);
#endif

  if (_index_update)
    {
      if ((_index_update + _index_bin[HLRC_myself]) > BINSIZE)
	{
	  fprintf (logfile, "Bin overflow\n");
	  fprintf (logfile, "Try increasing BINSIZE(currently %d)\n",
		   BINSIZE);
	  fflush (logfile);
	  HLRC_ErrPrint ("Bin overflow\n");
	  HLRC_ErrPrint ("Try increasing BINSIZE(currently %d)\n", BINSIZE);
	  exit (1);
	}

      _global_stamp[HLRC_myself]++;
      for (i = 0 ; i < _index_update; i++)
	{
	  relpage = _update_list[i];
	  home = _per_page[relpage].home;
	  if (home != HLRC_myself)
	    {

#ifdef INDIRECT_DIFFS

	      if (_per_page[relpage].twin == NULL)
		{
#ifdef HLRC_DEBUG
		  fprintf (logfile, "%d %d %d #\n", relpage, i,
			   index_update);
		  fflush (logfile);
#endif
		  continue;
		}

	      if (_per_page[relpage].dirty_and_invalid)
		{

		  _per_page[relpage].state = 0;
		  _per_page[relpage].dirty_and_invalid = 0;
#ifdef HLRC_DEBUG
		  fprintf (logfile, "End Interval: Dirty and Invalid[%d]\n",
			   relpage);
		  fflush (logfile);
#endif

		}
	      else
		{

		  _per_page[relpage].state = 1;
#ifdef NO_SWAP_READONLY_PAGE
		  add_mprotect (relpage, HLRC_PROT_READ_DISCARD_AT_SWAP);
#else /* NO_SWAP_READONLY_PAGE */
		  add_mprotect (relpage, HLRC_PROT_READ);
#endif /* NO_SWAP_READONLY_PAGE */
		}


#else /* INDIRECT_DIFFS */

	      _per_page[relpage].state = 1;

#ifdef NO_SWAP_READONLY_PAGE
	      add_mprotect (relpage, HLRC_PROT_READ_DISCARD_AT_SWAP);
#else /* NO_SWAP_READONLY_PAGE */
	      add_mprotect (relpage, HLRC_PROT_READ);
#endif /* NO_SWAP_READONLY_PAGE */

#endif /* INDIRECT_DIFFS */

	      offset = (diff_buff_size * _next_diff[home]) / WORDSIZE;

	      /* 
		 Wait for free diff buffer 
	      */
	      GETTIME (time1);
	      while (hlrc_protocol_control->diffBufferFree[home]
		     [_next_diff[home]] == 0);
	      GETTIME (time2);

	      HLRC_stat.diffwait += time2 - time1;
	      HLRC_EnterCriticalSection(0);
	      hlrc_protocol_control->diffBufferFree[home][_next_diff[home]] =
		0;
	      _diff_control[home][_next_diff[home]].page = relpage;
	      _diff_control[home][_next_diff[home]].tmst =
		_global_stamp[HLRC_myself];

	      /* if size == 0 should not send */
	      /* if full page , send special message */

#ifdef DIRECT_DIFFS
	      if (_per_page[relpage].twin == 0) {//cojvm
		//		while(1);
		fprintf(logfile,"_per_page[%i].twin == 0\n",relpage);fflush(logfile);
		hlrc_protocol_control->diffBufferFree[home][_next_diff[home]] = 1;
		HLRC_LeaveCriticalSection(0);
		goto fim;
	      }
	      _compute_and_send_diff (relpage,
				      (int *)((relpage+prot_first_shared_page)
					      <<logpagesize),
				      _per_page[relpage].size / WORDSIZE);
#else
     
	      _diff_control[home][_next_diff[home]].firstsize =
		_compute_diff (_per_page[relpage].twin,
			       (relpage + _first_shared_page) << logpagesize,
			       &_diff_buff_send[home][offset],
			       _per_page[relpage].size / WORDSIZE, &size);
	      _diff_control[home][_next_diff[home]].size = size;
	      if (size)
		{
		  HLRC_stat.msg_data++;
		  HLRC_stat.traffic_data += size * WORDSIZE;
		  HLRC_stat.traffic_diff += size * WORDSIZE;
		  HLRC_SendMessage (home, HLRC_myself,
				    offset * WORDSIZE,
				    (char *) &(_diff_buff_send[home][offset]),
				    size * WORDSIZE, DIFF_MESG, 0);
		}
#endif
	      offset = HLRC_DIFF_HDR_START (_next_diff[home]);
	      HLRC_SendRequest (home, HLRC_myself,
				offset * WORDSIZE,
				(volatile int *)
				&_diff_control[home][_next_diff[home]],
				sizeof (TDiffControl), DIFF);

	      _next_diff[home] = (_next_diff[home] + 1) % DIFF_WIN_SIZE;
#ifdef HLRC_DEBUG
	      fprintf (logfile, "free twin: %d\n", relpage);
	      fflush (logfile);
#endif
	      HLRC_LeaveCriticalSection(0);
	      HLRC_FreeTwin (relpage);

	    }
	  else
	    {
	      _per_page[relpage].state = HLRC_EXCL_STATE;
	    }
	fim:
	  _home_flush[home][relpage] =
	    _per_page[relpage].my_stamp[HLRC_myself] =
	    _global_stamp[HLRC_myself];

	  /*
	    Update _bin with the information of the modified page
	  */
	  if (i == 0)
	    {
	      previous = _index_bin[HLRC_myself];
	      _bin[HLRC_myself][_index_bin[HLRC_myself]++] = relpage;
	    }
	  else if (_bin[HLRC_myself][_index_bin[HLRC_myself] - 1] ==
		   (relpage - 1))
	    {
	      _bin[HLRC_myself][_index_bin[HLRC_myself]++] = -relpage;
	    }
	  else if (_bin[HLRC_myself][_index_bin[HLRC_myself] - 1] ==
		   (1 - relpage))
	    {
	      _bin[HLRC_myself][_index_bin[HLRC_myself] - 1] = -relpage;
	    }
	  else
	    {
	      _bin[HLRC_myself][_index_bin[HLRC_myself]++] = relpage;
	    }
	}
      _index_update = 0;



#ifdef HLRC_DEBUG
      fprintf (logfile, "\n");
      fflush (logfile);
#endif

      /* update the backward pointer */

#ifdef NO_SWAP_READONLY_PAGE
      add_mprotect (-1, HLRC_PROT_READ_DISCARD_AT_SWAP);
#else
      add_mprotect (-1, HLRC_PROT_READ);
#endif

      _bin[HLRC_myself][_index_bin[HLRC_myself]++] = previous;
    }

  HLRC_LeaveCriticalSection (1);

#ifdef HLRC_DEBUG
  fprintf (logfile, "end_interval end\n");
  fflush (logfile);
#endif
}


/*
  _send_page:
  Send page to remote node in response to a PAGE request.
  The page is sent, followed by a control message which 
  contains the version of the page.
*/
void
_send_page (int to, int relpage)
{
  int j;
  int error;
  int reg_part, part_offset;
  int start_page;

#ifdef HLRC_DEBUG
  fprintf (logfile, "[%d] Sending page: %d to %d\n", (int)pthread_self (), relpage,
	   to);
  fflush (logfile);
#endif

  for (j = 0; j < HLRC_nprocs; j++)
    {
      HLRC_scratch_buffer[HLRC_myself][to][msg_tmst1_start + j] =
	_home_flush[j][relpage];
    }

  HLRC_stat.msg_control++;
  HLRC_stat.traffic_control += (NPROCS + MSG_HDR_LEN) * WORDSIZE;
  HLRC_stat.msg_data++;
  HLRC_stat.traffic_data += _per_page[relpage].size;

#ifdef PROT_HANDLE
  start_page = prot_first_shared_page;
#else
  start_page = _first_shared_page;
#endif /* PROT_HANDLE */

#ifdef COPY_ON_RECV

#ifdef COPY_ON_SEND
  bcopy ((relpage + start_page) << logpagesize, HLRC_page_send_buff[to],
	 _per_page[relpage].size);
  HLRC_SendMessage (to, HLRC_myself, 0,
		    HLRC_page_send_buff[to],
		    _per_page[relpage].size, PAGE_MESG, 0);

#else /* COPY_ON_SEND */
  reg_part = relpage / MAX_REGMEM_PAGES;
  HLRC_SendMessage (to, HLRC_myself, 0,
		    (char *) ((relpage + start_page) << logpagesize),
		    _per_page[relpage].size, PAGE_MESG, reg_part);

#endif

#else /* COPY_ON_RECV */

#ifndef COPY_ON_SEND

  reg_part = relpage / MAX_REGMEM_PAGES;
  part_offset = relpage % MAX_REGMEM_PAGES;

  HLRC_SendMessage (to, HLRC_myself,
		    part_offset << logpagesize,
		    (char *) ((relpage + start_page) << logpagesize),
		    _per_page[relpage].size, PAGE_MESG, reg_part);
#endif

  /* Initialize HLRC_scratch_buffer[][] ? */
  HLRC_scratch_buffer[HLRC_myself][to][msg_type_index] = PAGE;
  HLRC_scratch_buffer[HLRC_myself][to][msg_node_index] =
    _per_page[relpage].home;
  HLRC_scratch_buffer[HLRC_myself][to][msg_item_index] = relpage;

  /*
    Renounce exclusive ownership and change state to READONLY
  */
  if (_per_page[relpage].state == -HLRC_EXCL_STATE)
    {
      _per_page[relpage].state = -HLRC_PROT_READ;
    }
  if (_per_page[relpage].state == HLRC_EXCL_STATE)
    {
      _per_page[relpage].state = HLRC_PROT_READ;
      if (error =
	  mprotect ((void *) ((relpage + _first_shared_page) << logpagesize),
		    pagesize, HLRC_PROT_READ))
	{
	  fprintf (logfile, "mprotect error: 1\n");
	  fflush (logfile);
	  perror ("mprotect error");
	  exit (1);
	}
    }

  HLRC_SendMessage (to, to, (msg_tmst1_start) * WORDSIZE,
		    (char *)
		    &HLRC_scratch_buffer[HLRC_myself][to][msg_tmst1_start],
		    (NPROCS + MSG_HDR_LEN) * WORDSIZE, REQUEST_MESG, 0);

#ifdef HLRC_DEBUG
  fprintf (logfile, "Done sending page number: %d\n", relpage);
  fflush (logfile);
#endif
}


/*
  _send_lock:
  Send lock to remote node in response to LOCK request.
  The write notices are sent, followed by a control
  message which determines the current snapshot of
  the shared data determined by this lock.

*/

void
_send_lock (int flag, int lock, int to, volatile int *stamp)
{
  register int i, j, k, index;

#ifdef HLRC_DEBUG
  fprintf (logfile, "Sending lock: %d to %d\n", lock, to);
  fflush (logfile);
#endif

  _lock_data[lock].state = 0;

  if (_lock_data[lock].stamp[HLRC_myself] > _last_stamp[to])
    {
      _last_stamp[to] = _lock_data[lock].stamp[HLRC_myself];
    }

  if (stamp[HLRC_myself] > _last_stamp[to])
    {
      _last_stamp[to] = stamp[HLRC_myself];
    }

  /* send lock with bins and tmsp */

  for (j = 0; j < HLRC_nprocs; j++)
    {
      if (_lock_data[lock].stamp[j] > stamp[j])
	{
	  for (k = 0, index = _lock_data[lock].index[j];
	       k < (_lock_data[lock].stamp[j] - stamp[j]); k++)
	    {
	      if (_bin[j][index - 1] < 0) break;
	      index = _bin[j][index - 1];
	    }
	  HLRC_stat.msg_control++;
	  HLRC_stat.traffic_control +=
	    (_lock_data[lock].index[j] - index) * WORDSIZE;
	  HLRC_SendMessage (to, j, index * WORDSIZE,
			    (char *) &_bin[j][index],
			    (_lock_data[lock].index[j] - index) * WORDSIZE,
			    BIN_MESG, 0);
	}
    }

  HLRC_scratch_buffer[HLRC_myself][to][msg_type_index] = LOCK;
  HLRC_scratch_buffer[HLRC_myself][to][msg_node_index] = lock % HLRC_nprocs;
  HLRC_scratch_buffer[HLRC_myself][to][msg_item_index] = lock;

  bcopy ((const void *)_lock_data[lock].index,
	 (void *)&HLRC_scratch_buffer[HLRC_myself][to][msg_tmst2_start],
	 WORDSIZE * NPROCS);
  bcopy ((const void *)_lock_data[lock].stamp,
	 (void *)&HLRC_scratch_buffer[HLRC_myself][to][msg_tmst1_start],
	 WORDSIZE * NPROCS);

  HLRC_stat.msg_control++;
  HLRC_stat.traffic_control += (2 * NPROCS + MSG_HDR_LEN) * WORDSIZE;

  HLRC_SendMessage (to, to, (msg_tmst2_start) * WORDSIZE,
		    (char *)
		    &HLRC_scratch_buffer[HLRC_myself][to][msg_tmst2_start],
		    (2 * NPROCS + MSG_HDR_LEN) * WORDSIZE, REQUEST_MESG, 0);
#ifdef HLRC_DEBUG
  fprintf (logfile, "Done sending lock: %d\n", lock);
  fflush (logfile);
#endif
}
/*
  _apply_wn: 
  Apply the write notices(available in the _bin buffer) 
  to generate invalidations. 
  _index_bin[i] is an index into the _bin structure and 
  points to the top of _bin[i]. The element index_bin[i]-1 
  contains a pointer to the index_bin[i] for the previous 
  interval. As for the write notices, the space used is
  optimized by storing the dirty page ids in terms of
  ranges, and not as individual page ids.
*/

void
_apply_wn (int bindex, int tmst, int i)
{
  register int j, k;
  int index, oindex;
  register int relpage;
  int size;

#ifdef HLRC_DEBUG
  fprintf (logfile, "_apply_wn - node: %d tmst: %d _global_stamp=%d\n", i, tmst, _global_stamp[i]);
  fflush (logfile);
#endif

  /* assume the same as for PAGE for now */
  if (tmst > _global_stamp[i])
    {
      if (bindex > _index_bin[i])
	_index_bin[i] = bindex;
#ifdef HLRC_DEBUG
      fprintf (logfile, "index_bin: %d ", bindex);
      fflush (logfile);
#endif
      for (j = 0, oindex = _index_bin[i]; j < (tmst - _global_stamp[i]);
	   j++, oindex = index)
	{
	  size = oindex - _bin[i][oindex - 1] - 1;
#ifdef HLRC_DEBUG
	  fprintf (logfile, "size: %d oindex: %d _bin: %d j: %d ", size, oindex,
		   _bin[i][oindex - 1],j);
	  fflush (logfile);
#endif
	  index = _bin[i][oindex - 1];
	  for (k = index; k < (index + size); k++)
	    {
	      if ((relpage = _bin[i][k]) >= 0)
		{
		  if (_per_page[relpage].home == HLRC_myself)
		    {
		      if (_home_flush[i][relpage] < (tmst - j))
			{
			  if (_per_page[relpage].my_stamp[i] < (tmst - j))
			    {
			      _per_page[relpage].my_stamp[i] = tmst - j;
			    }
			  if (_per_page[relpage].state > 0)
			    {
			      _per_page[relpage].state =
				-_per_page[relpage].state;
			      add_mprotect (relpage, HLRC_PROT_NONE);
			    }
			}
		    }
		  else
		    {
		      if (_per_page[relpage].state)
			{
			  if (_per_page[relpage].my_stamp[i] < (tmst - j))
			    {
#ifdef INDIRECT_DIFFS
			      if (_per_page[relpage].state > 1)
				{
#ifdef HLRC_DEBUG
				  fprintf (logfile, "Dirty and Invalid[%d]\n",
					   relpage);
				  fflush (logfile);
#endif
				  _per_page[relpage].dirty_and_invalid = 1;
				}
#endif
			      _per_page[relpage].my_stamp[i] = tmst - j;
			      _per_page[relpage].state = 0;
#ifdef HLRC_DEBUG
			      fprintf (logfile, "relpage %i state = %i\n", relpage, _per_page[relpage].state);
			      fflush (logfile);
#endif
			      add_mprotect (relpage, HLRC_PROT_NONE);
			    }
			}
		      else if (_per_page[relpage].my_stamp[i] < (tmst - j))
			{
			  _per_page[relpage].my_stamp[i] = tmst - j;
			}
		    }
		}
	      else
		for (relpage = _bin[i][k - 1] + 1; relpage <= (-_bin[i][k]);
		     relpage++)
		  {
		    if (_per_page[relpage].home == HLRC_myself)
		      {
			if (_home_flush[i][relpage] < (tmst - j))
			  {
			    if (_per_page[relpage].my_stamp[i] < (tmst - j))
			      {
				_per_page[relpage].my_stamp[i] = tmst - j;
			      }
			    if (_per_page[relpage].state > 0)
			      {
				_per_page[relpage].state =
				  -_per_page[relpage].state;
				add_mprotect (relpage, HLRC_PROT_NONE);
			      }
			  }
		      }
		    else
		      {
			if (_per_page[relpage].state)
			  {
			    if (_per_page[relpage].my_stamp[i] < (tmst - j))
			      {
#ifdef INDIRECT_DIFFS
				if (_per_page[relpage].state > 1)
				  {
#ifdef HLRC_DEBUG
				    fprintf (logfile,
					     "Dirty and Invalid[%d]\n",
					     relpage);
				    fflush (logfile);
#endif
				    _per_page[relpage].dirty_and_invalid = 1;
				  }
#endif
				_per_page[relpage].my_stamp[i] = tmst - j;

				/* optimize mprotect later */
				_per_page[relpage].state = 0;
				add_mprotect (relpage, HLRC_PROT_NONE);
			      }
			  }
			else if (_per_page[relpage].my_stamp[i] < (tmst - j))
			  {
			    _per_page[relpage].my_stamp[i] = tmst - j;
			  }
		      }
		  }
#ifdef HLRC_DEBUG
	      fprintf (logfile, ", my_stamp: %d\n", _per_page[relpage].my_stamp[i]);
	      fflush (logfile);
#endif
	    }
	}
      _global_stamp[i] = tmst;
      add_mprotect (-1, HLRC_PROT_NONE);
    }
}


/*
  _compute_and_send_diff: 
  Compute the DIFF between the modified page and the twin and 
  return the size of the DIFF
*/
#ifdef DIRECT_DIFFS
void
_compute_and_send_diff (int relpage, int *new, int size)
{
  register int i;
  int next = 0;
  int firstlength = 0;
  int length = 0;
  int last;
  int start = -1;
  int *old = _per_page[relpage].twin;
  HLRC_stat.diff_created++;
#ifdef HLRC_DEBUG

  fprintf (logfile,
	   "Sending DIFF for page %i (old: %p new: %p)\n", relpage, old, new);
  fflush (logfile);
#endif

  if (old == NULL || new == NULL) {
    int a=1;

#ifdef HLRC_DEBUG

    fprintf (logfile,
	     "Error: null par in call\n");
    fflush (logfile);
#endif
    while (a);

  }

  for (i = 0; i < size; i++)
    {
      if (old[i] != new[i])
	{
	  if (length == 0)
	    start = i;
	  length++;
	}
      else
	{
	  if (length > 0)
	    {
	      int reg_part = relpage / MAX_REGMEM_PAGES;
	      int part_offset = relpage % MAX_REGMEM_PAGES;
	      HLRC_SendDiff (next, _per_page[relpage].home, HLRC_myself,
			     (part_offset << logpagesize) +
			     (start * WORDSIZE), (char *) (&new[start]),
			     length * WORDSIZE, DIFF_MESG, reg_part);
	      next = (next + 1) % DIFF_MAX_SEND;
	      length = 0;
	    }
	}
    }

  if (length)
    {
      int reg_part = relpage / MAX_REGMEM_PAGES;
      int part_offset = relpage % MAX_REGMEM_PAGES;
      HLRC_SendDiff (next, _per_page[relpage].home, HLRC_myself,
		     (part_offset << logpagesize) + (start * WORDSIZE),
		     (char *) (&new[start]), length * WORDSIZE,
		     DIFF_MESG, reg_part);
      next = (next + 1) % DIFF_MAX_SEND;
    }

}

#else /* DIRECT_DIFFS */

_compute_diff (int *old, int *new, int *to, int size, int *retsize)
{
  register int i;
  int next = 0;
  int firstlength = 0;
  int length = 0;
  int last;
  int start;

  HLRC_stat.diff_created++;

  for (i = 0; i < size; i++)
    {
      if (old[i] != new[i])
	{
	  if (length == 0)
	    start = i;
	  length++;
	  to[next++] = new[i];
	}
      else
	{
	  if (length > 0)
	    {
	      if (firstlength == 0)
		{
		  firstlength = (start << 16) | length;
		  last = next++;
		}
	      else
		{
		  to[last] = (start << 16) | length;
		  last = next++;
		}

	      length = 0;
	    }
	}
    }

  if (length)
    {
      if (firstlength == 0)
	{
	  firstlength = (start << 16) | length;
	}
      else
	{
	  to[last] = (start << 16) | length;
	}
    }
  else if (firstlength)
    {
      next--;
    }

  *retsize = next;

  return firstlength;
}


/*
  _apply_diff: 
  Apply the DIFF to the page in shared memory at the home 
  of the page
*/
_apply_diff (volatile int *diff, volatile int *page, int firstsize, int size)
{
  register int i;
  int length;
  int start;
  int current;
  int index;
  for (i = 0, current = firstsize; i < size;)
    {
      length = current & 0xffff;
      start = current >> 16;
      bcopy (&diff[i], &page[start], length * WORDSIZE);
#ifdef HLRC_DEBUG
      for (index=0;index<size;index+=sizeof(int))
	fprintf (logfile, "diff: %i ", (int)diff[index]);
      fflush (logfile);
#endif
      i += length;
      if (i < size)
	{
	  current = diff[i++];
	}
    }

  HLRC_stat.diff_applied++;

}
#endif /* DIRECT_DIFFS */


/*
  HLRC_AllocTwin:
  Allocate memory for twin page. Try to reuse previously
  allocated memory.
*/
int *
HLRC_AllocTwin ()
{
  if (_notwins > 0)
    {
      --_notwins;
      return (int *) _free_twins[_notwins];
    }
  else
    {
      return (int *) _AMALLOC (pagesize, pagesize);
    }
}


void
HLRC_cleanup (int dummy, struct sigcontext_struct scs)
{
  HLRC_ReleaseResources ();

  HLRC_ErrPrint ("Done releasing resources\n");

  pthread_exit (0);

  exit (0);
}

/*
  HLRC_HandleRequests:
  Thread to handle all incoming communication. This thread
  blocks on the receive CQ till a message is received, and 
  processes the message as soon as it unblocks from the wait.
*/

void *
HLRC_HandleRequests (void *dummy_arg){
  unsigned j, k, req_type,i;
  void *ptr;
  int home, last;
  int offset;
  int offset_in_words;
  unsigned msg_type, from, relpage;
  int nodeId, bufId;
  int sender, senderIndex;
  unsigned long long time1, time2, time3, time4;
  TDiffControl *diff_recv = NULL;
  int error;
  struct sigaction HLRC_sig;
  recv_t ret_param, ret_ctrl;


  /*  HLRC_sig.sa_handler = (void (*)(int)) HLRC_cleanup;
      HLRC_sig.sa_flags = 0;*/

  sigaction (SIGTERM, &HLRC_sig, 0);

#ifdef PROC_BIND_SUPPORT
  handler_cpu = curr_cpu;
  if (sysmp (MP_MUSTRUN, curr_cpu) < 0)
    {
      perror ("Binding to the current processor\n");
      HLRC_ErrPrint ("Binding to the current processor(%d)\n", curr_cpu);
      exit (1);
    }
#endif
  msg_type = 0;

  while (msg_type != END_MESG){
    if (TRANS_BlockForReceive(&ret_param, &ret_ctrl) != TRANS_SUCCESS){
      fprintf (logfile, "Error in BlockForReceive\n");
      fflush (logfile);
      HLRC_ErrPrint ("Error in BlockForReceive\n");
      exit (1);
    }

    nodeId = ret_param.nodeId;
    bufId = ret_param.bufId;
    sender = ret_param.senderId;
    senderIndex = ret_param.index;
    msg_type = ret_param.mesgType;
    relpage = -1;
    from = -1;

#ifdef HLRC_DEBUG
    fprintf (logfile, "Received from node: %d, type: %i (", nodeId, msg_type);
    switch (msg_type){
    case PAGE:
      fprintf (logfile, "PAGE ");
      break;
    case LOCK:
      fprintf (logfile, "LOCK ");
      break;
    case DIFF:
      fprintf (logfile, "DIFF ");
      break;
    }
    fprintf (logfile, ") bufId: %d(%d,%d)\n", bufId, sender, senderIndex);
    fflush (logfile);
#endif

    HLRC_stat.requests_recv++;

    /*
      Extract the control information based on the type of the
      message
    */

    if (msg_type == DIFF){
      diff_recv =
	(TDiffControl *) &
	HLRC_request_buffer[nodeId][HLRC_DIFF_HDR_START (bufId)];
      msg_type = diff_recv->cmd;
      relpage = diff_recv->page;
      from = diff_recv->proc;
      fflush(logfile);
    }
    else 
      {
	switch (msg_type){
	case PAGE:
	  msg_type = HLRC_request_buffer[nodeId][msg_type_index];
	case PAGE_MESG:
	case LOCK:
	  relpage = HLRC_request_buffer[nodeId][msg_item_index];
	  from = HLRC_request_buffer[nodeId][msg_node_index];
	  fflush(logfile);
	  break;
	}
      }

#ifdef HLRC_DEBUG
    fprintf (logfile, "Inside HLRC_Handler: Mesg: %d for: %d from: %d\n",
	     msg_type, relpage, from);
    fflush (logfile);
#endif
    switch (msg_type)
      {
      case BARRIER_MESG:
	sem_wait (&mutex_barrier);
	sem_post (&sem_barrier);  // libera a thread da aplicacao
	fila[fim_fila] = sender;
	fim_fila = (fim_fila + 1) % HLRC_nprocs;
	tam_fila++;
	sem_post (&mutex_barrier);
	break;
      case END_MESG:
	break;
      case REQUEST_MESG:
	req_type = HLRC_request_buffer[HLRC_myself][msg_type_index];
	switch (req_type)
	  {
	  case LOCK:
	    sem_post (&sem_lock); // libera a thread da aplicacao
	    break;
	  case PAGE:
	    sem_post (&sem_page); // libera a thread da aplicacao
	    break;
	  }
	break;
      case DIFF_MESG:
	break;
      case DIFFACK_MESG: case PAGE_MESG: case BIN_MESG:
	break;
      case BARR_END_MESG:
	sem_post(&sem_barr_end);
	break;
      case DIFF:

#ifndef PROT_HANDLE

	offset_in_words = (diff_buff_size * diff_recv->no) / WORDSIZE;
#ifdef HLRC_DEBUG
	fprintf (logfile, "offset_in_words: %d\n", offset_in_words);
	fflush (logfile);
#endif

	if (_per_page[relpage].state < HLRC_PROT_RW)
	  {
	    if (error =
		mprotect ((relpage + _first_shared_page) << logpagesize,
			  pagesize, HLRC_PROT_RW))
	      {
		perror ("mprotect error");
		exit (1);
	      }
	    _apply_diff (&_diff_buff_recv[from][offset_in_words],
			 (int *) ((relpage + _first_shared_page) <<
				  logpagesize), diff_recv->firstsize,
			 diff_recv->size);

	    if (_per_page[relpage].state < HLRC_PROT_READ)
	      {
		if (error =
		    mprotect ((relpage + _first_shared_page) << logpagesize,
			      pagesize, HLRC_PROT_NONE))
		  {
		    perror ("mprotect error");
		    exit (1);
		  }
	      }
	    else if (error =
		     mprotect ((relpage +
				_first_shared_page) << logpagesize,
			       pagesize, HLRC_PROT_READ))
	      {
		perror ("mprotect error");
		exit (1);
	      }

	  }
	else
	  {
	    _apply_diff (&_diff_buff_recv[from][offset_in_words],
			 (int *) ((relpage + _first_shared_page) <<
				  logpagesize), diff_recv->firstsize,
			 diff_recv->size);
	  }

#else /* PROT_HANDLE */

#ifdef DIRECT_DIFFS
	diff_recv->no = (diff_recv->no) % DIFF_WIN_SIZE;
#else
	offset_in_words = (diff_buff_size * diff_recv->no) / WORDSIZE;
#ifdef HLRC_DEBUG
	fprintf (logfile, "offset_in_words: %d\n", offset_in_words);
	fflush (logfile);
#endif
	_apply_diff (&_diff_buff_recv[from][offset_in_words],
		     (int *) ((relpage + prot_first_shared_page) <<
			      logpagesize), diff_recv->firstsize,
		     diff_recv->size);

#endif

#endif /* PROT_HANDLE */

	//(volatile)_home_flush[nodeId][relpage] = diff_recv->tmst; Anderson

	_home_flush[nodeId][relpage] = diff_recv->tmst;

	HLRC_stat.msg_control++;
	HLRC_stat.traffic_control += WORDSIZE;

	offset =
	  ((long)
	   &(hlrc_protocol_control->diffBufferFree[HLRC_myself][bufId])) -
	  ((long) hlrc_protocol_control);

	HLRC_SendMessage (from, HLRC_myself, offset,
			  (char *) diffAck, WORDSIZE, DIFFACK_MESG, 0);

	/*
	  If this DIFF generates a new version of the page for
	  which there is a pending request, satisfy the request
	*/
	for (j = 0; j < HLRC_nprocs; j++)
	  {
	    if ((j != HLRC_myself) && (_pending[j].type == PAGE) &&
		(_pending[j].addr == relpage))
	      {

		for (k = 0; k < HLRC_nprocs; k++)
		  {
		    if (_home_flush[k][relpage] < _pending[j].stamp[k])
		      break;
		  }

		if (k != HLRC_nprocs)
		  continue;

		_pending[j].type = 0;

		_send_page (j, relpage);

	      }
	  }

	break;

      case PAGE:
	if (_per_page[relpage].home == -1)
	  {

	    if ((relpage % HLRC_nprocs) == HLRC_myself)
	      {
		/*
		  At the default home:
		  If the page doesn't yet have a home,
		  make the requestor, home of the page
		*/
		_per_page[relpage].home = from;
		_per_page[relpage].state = 0;
#ifdef HLRC_DEBUG
		fprintf (logfile, "%i is home for page: %d (state: %i)\n", from, relpage, _per_page[relpage].state);
		fflush (logfile);
#endif
	      }
	    else
	      {
		/*
		  At other nodes:
		  If the page doesn't yet have a home,
		  since I have been contacted for the 
		  page, I am the home.
		*/
		_per_page[relpage].home = HLRC_myself;
#ifdef HLRC_DEBUG
		fprintf (logfile, "I am home for page: %d\n", relpage);
		fflush (logfile);
#endif
	      }

	  }
	else if (_per_page[relpage].home != HLRC_myself)
	  {
	    HLRC_stat.forward++;

	    home = _per_page[relpage].home;
	    /*
	      Copy HLRC_request_buff onto HLRC_scratch_buff and send
	    */
	    memcpy ((void*)&HLRC_scratch_buffer[from][home][msg_tmst1_start],
		    (const void *)&HLRC_request_buffer[from][msg_tmst1_start],
		    sizeof (int) * (NPROCS + MSG_HDR_LEN));

#ifdef HLRC_DEBUG
	    fprintf (logfile, "Forwarding PAGE request to %d\n", home);
	    fflush (logfile);
#endif

	    HLRC_SendRequest (home, from,
			      (msg_tmst1_start) * WORDSIZE,
			      &HLRC_scratch_buffer[from][home]
			      [msg_tmst1_start],
			      (NPROCS + MSG_HDR_LEN) * sizeof (int), PAGE);

	    break;
	  }

	/* 
	   Check the tmsp to see if the requested version
	   of the page is available
	*/

	for (j = 0; j < HLRC_nprocs; j++)
	  {
	    if (_home_flush[j][relpage] <
		HLRC_request_buffer[from][msg_tmst1_start + j])
	      {
		break;
	      }
	  }

	if (j == HLRC_nprocs)
	  {
	    _send_page (from, relpage);
      
	  }
	else
	  {
	    _pending[from].type = PAGE;
	    _pending[from].addr = relpage;
	    //#ifdef HLRC_DEBUG
	    fprintf (logfile, "Pending PAGE(%d) request: [%d] %d, %d\n",
		     relpage, j, _home_flush[j][relpage],
		     HLRC_request_buffer[from][msg_tmst1_start + j]);
	    fflush (logfile);
	    //#endif
	    bcopy ((const void *)&HLRC_request_buffer[from][msg_tmst1_start],
		   (void *)_pending[from].stamp, WORDSIZE * NPROCS);
	  }
	break;

      case LOCK:

#ifdef HLRC_DEBUG
	fprintf (logfile, "[%d] Processing LOCK request to %d\n",
		 (int)pthread_self (), from);
	fflush (logfile);
#endif

	if ((relpage % HLRC_nprocs) == HLRC_myself)
	  {
	    if (_lock_data[relpage].last == HLRC_myself)
	      {

		_lock_data[relpage].last = from;
		if (_lock_data[relpage].state)
		  {
		    _send_lock (SERVER_THREAD, relpage, from,
				&HLRC_request_buffer[from]
				[msg_tmst1_start]);
		  }
		else
		  {
		    _lock_data[relpage].state = 1;
		    _pending[from].type = LOCK;
		    _pending[from].addr = relpage;
#ifdef HLRC_DEBUG
		    fprintf (logfile, "[%d] %d must wait: lock pending\n", (int)pthread_self (),from);
		    fflush (logfile);
#endif
		    bcopy ((const void *)&HLRC_request_buffer[from][msg_tmst1_start],
			   (void *)_pending[from].stamp, WORDSIZE * NPROCS);
		  }

	      }
	    else
	      {
		/* 
		   Forward_ to last  
		*/
		if (HLRC_request_buffer[from][msg_tmst1_start + HLRC_myself]
		    > _last_stamp[from])
		  {
		    _last_stamp[from] =
		      HLRC_request_buffer[from][msg_tmst1_start +
						HLRC_myself];
		  }
#ifdef HLRC_DEBUG
		fprintf (logfile, "[%d] lock=> from: %d last: %d lock_data.last: %d\n", (int)pthread_self (),from,last, _lock_data[relpage].last);
		fflush (logfile);
#endif
		if (_lock_data[relpage].last != from) {
		  last = _lock_data[relpage].last;
		  /*
		    Copy HLRC_request_buff onto HLRC_scratch_buff and send
		  */
		  memcpy ((void *)&HLRC_scratch_buffer[from][last][msg_tmst1_start],
			  (const void *)&HLRC_request_buffer[from][msg_tmst1_start],
			  sizeof (int) * (NPROCS + MSG_HDR_LEN));

#ifdef HLRC_DEBUG
		  fprintf (logfile, "Forwarding LOCK request to %d\n", last);
		  fflush (logfile);
#endif

		  HLRC_SendRequest (last, from,
				    (msg_tmst1_start) * WORDSIZE,
				    &HLRC_scratch_buffer[from][last]
				    [msg_tmst1_start],
				    (NPROCS + MSG_HDR_LEN) * sizeof (int),
				    LOCK);
		  _lock_data[relpage].last = from;
		} else {
		  _send_lock (SERVER_THREAD, relpage, from,
			      &HLRC_request_buffer[from]
			      [msg_tmst1_start]);
		}

		break;
	      }
	  }
	else
	  {

	    /*
	      If lock is available, send the lock
	    */
	    if (_lock_data[relpage].state)
	      {
		_send_lock (SERVER_THREAD, relpage, from,
			    &HLRC_request_buffer[from][msg_tmst1_start]);
	      }
	    else
	      {
		_lock_data[relpage].state = 1;
		_pending[from].type = LOCK;
		_pending[from].addr = relpage;

#ifdef HLRC_DEBUG
		fprintf (logfile, "[%d] %d must wait: lock pending\n", (int)pthread_self (),from);
		fflush (logfile);
#endif

		bcopy ((const void *)&HLRC_request_buffer[from][msg_tmst1_start],
		       (void *)_pending[from].stamp, WORDSIZE * NPROCS);
	      }
	  }
#ifdef HLRC_DEBUG
	fprintf (logfile, "[%d] done with LOCK request\n", (int)pthread_self ());
	fflush (logfile);
#endif
	break;
      case WAIT:
#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] WAIT request\n", HLRC_myself);
	fflush(logfile);
#endif

	_lock_data[relpage].wait[from] = 1;

#ifdef HLRC_DEBUG
	fprintf(logfile,
		"[%i] Proc inserido na fila _lock_data[%i].wait[%i] = 1\n",
		HLRC_myself, relpage, from);
	fflush(logfile);
#endif
	break;

      case TIMEDOUT:
#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] TIMEDOUT request\n", HLRC_myself);
	fflush(logfile);
#endif

	_lock_data[relpage].wait[from] = 0;

#ifdef HLRC_DEBUG
	fprintf(logfile,
		"[%i] Proc retirado da fila lock:%d proc:%i\n",
		HLRC_myself, relpage, from);
	fflush(logfile);
#endif
	break;

      case NOSLEEP:
#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] NOSLEEP request\n", HLRC_myself);
	fflush(logfile);
#endif

	pthread_cond_signal(&cond_);

#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] Proc acordado.\n", HLRC_myself);
	fflush(logfile);
#endif
	break;


      case SIGNAL:
#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] SIGNAL request %d\n", HLRC_myself,
		relpage);
	fflush(logfile);
#endif
	/* Verificando se existe alguem esperando por sinal neste lock */
	for (i = 0; i < NPROCS; i++) {

	  /* existe thread esperando sinal */
	  if (_lock_data[relpage].wait[i]) {

	    if (i == HLRC_myself) {
	      /* thread bloqueada esta no processador local */
#ifdef BUSY_WAIT
	      HLRC_request_buffer[HLRC_myself][msg_type_index] = SIGNAL;	// acordar app
#else
	      pthread_cond_signal(&cond_);
#endif

#ifdef HLRC_DEBUG
	      fprintf(logfile,
		      "[%i] acordando thread do proc local\n",
		      HLRC_myself); fflush(logfile);
#endif
	    } else {
	      /* Envio msg para o desbloquear thread remota */
#ifdef BUSY_WAIT
	      HLRC_scratch_buffer[HLRC_myself][i][msg_type_index]
		= SIGNAL;
#else
	      HLRC_scratch_buffer[HLRC_myself][i][msg_type_index]
		= NOSLEEP;
#endif
	      HLRC_scratch_buffer[HLRC_myself][i][msg_node_index]
		= HLRC_myself;
	      HLRC_scratch_buffer[HLRC_myself][i][msg_item_index]
		= relpage;

#ifdef BUSY_WAIT
	      HLRC_SendMessage(i, i, (msg_tmst1_start) * WORDSIZE,
			       (char *)&HLRC_scratch_buffer[HLRC_myself]
			       [i][msg_tmst1_start],
			       (NPROCS + MSG_HDR_LEN) * WORDSIZE,
			       REQUEST_MESG,0);
#else
	      HLRC_SendRequest(i, HLRC_myself,
			       (msg_tmst1_start) * WORDSIZE,
			       &HLRC_scratch_buffer[HLRC_myself]
			       [i][msg_tmst1_start],
			       (NPROCS +MSG_HDR_LEN) * sizeof(int),
			       NOSLEEP);
#endif
#ifdef HLRC_DEBUG
	      fprintf(logfile,
		      "[%i] HLRC_SendMessage (acordando thread remota) dest:%i\n",
		      HLRC_myself, i);
	      fflush(logfile);
#endif
	    }

	    /* retirando thread da fila de espera */
	    _lock_data[relpage].wait[i] = 0;

	    break;
	  }

	}

#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] terminando signal\n", HLRC_myself);
	fflush(logfile);
#endif
	break;

      case BROADCAST:
#ifdef HLRC_DEBUG
	fprintf(logfile, "[%i] BROADCAST request %d\n", HLRC_myself,
		relpage); fflush(logfile);
#endif
	/* Verificando se existe alguem esperando por sinal neste lock */
	for (i = 0; i < NPROCS; i++) {

	  /* existe thread esperando sinal */
	  if (_lock_data[relpage].wait[i]) {

	    if (i == HLRC_myself) {
	      /* thread bloqueada esta no processador local */
#ifdef BUSY_WAIT
	      HLRC_request_buffer[HLRC_myself][msg_type_index] = SIGNAL;	// acordar app
#else
	      pthread_cond_signal(&cond_);
#endif
#ifdef HLRC_DEBUG
	      fprintf(logfile,
		      "[%i] acordando thread do proc local\n",
		      HLRC_myself); fflush(logfile);
#endif
	    } else {
	      /* Envio msg para o desbloquear thread remota */
#ifdef BUSY_WAIT
	      HLRC_scratch_buffer[HLRC_myself][i][msg_type_index]
		= SIGNAL;
#else
	      HLRC_scratch_buffer[HLRC_myself][i][msg_type_index]
		= NOSLEEP;
#endif
	      HLRC_scratch_buffer[HLRC_myself][i][msg_node_index]
		= HLRC_myself;
	      HLRC_scratch_buffer[HLRC_myself][i][msg_item_index]
		= relpage;

#ifdef BUSY_WAIT
	      HLRC_SendMessage(i, i,
			       (msg_tmst1_start) * WORDSIZE,
			       (char *)
			       &HLRC_scratch_buffer[HLRC_myself]
			       [i][msg_tmst1_start],
			       (NPROCS + MSG_HDR_LEN) * WORDSIZE,
			       REQUEST_MESG,0);
#else
	      HLRC_SendRequest(i, HLRC_myself,
			       (msg_tmst1_start) * WORDSIZE,
			       &HLRC_scratch_buffer[HLRC_myself]
			       [i][msg_tmst1_start],
			       (NPROCS +
				MSG_HDR_LEN) * sizeof(int),
			       NOSLEEP);
#endif
#ifdef HLRC_DEBUG
	      fprintf(logfile,
		      "[%i] HLRC_SendMessage (acordando thread remota) dest:%i\n",
		      HLRC_myself, i);
	      fflush(logfile);
#endif

	    }

	    /* retirando thread da fila de espera */
	    _lock_data[relpage].wait[i] = 0;

	  }
	}
	break;

      default:
	fprintf (logfile, "Unknown request %d\n", msg_type);
	fflush (logfile);
	HLRC_ErrPrint ("Unknown request %d\n", msg_type);
	exit (1);
      }

    HLRC_LeaveCriticalSection (0);
#if 0
    curr_cpu = sysmp (MP_CURPROC);
    if (curr_cpu != handler_cpu)
      {
	fprintf (logfile, "Switched CPU\n");
	fflush (logfile);
	exit (1);
      }
#endif

  }

}
/*
  HLRC_Init: Initialization routine for the HLRC protocol
*/
void
HLRC_Init (int argc, char *argv[])
{
  char abspathname[1024];
  char zerobuf[4096];
  register int i, j;
  register int first;
  int *old_first_alloc;
  int serverNodes = 0;
  int remapmemory=0, *remappointer;
  unsigned long l;

#ifdef LOSE_TTY
  {
    int null_fd;

    if ((null_fd =
	 open ("/tmp/ttyfile", O_RDWR | O_CREAT | O_TRUNC, S_IRWXU)) == -1)
      {
	fprintf (stderr, "Cannot open %s", "/tmp/ttyfile");
	exit (1);
      }

    dup2 (null_fd, 0);
    dup2 (null_fd, 1);
    dup2 (null_fd, 2);
    close (null_fd);
    if (fork ())
      _exit (0);
    setsid ();
    if (fork ())
      _exit (0);
  }
#endif

  if(!cpu_frequency()) {
    fprintf (stderr, "Error reading CPU Frequency in /proc/cpuinfo");
    exit (1);
  }

  logfile = stderr;
  sem_init (&sem_barrier, 0, 0);
  sem_init (&mutex_barrier, 0, 1);
  sem_init (&mutex_comm, 0, 1);
  sem_init (&sem_lock, 0, 0);
  sem_init (&sem_page, 0, 0);
  sem_init (&sem_barr_end,0, 0);
  sem_init (&mutex_barr_end, 0, 1);
  in_fila = fim_fila = tam_fila = 0;

  /* 
     Get the hostname and store it in HLRC_myname
  */
  gethostname (HLRC_myname_short, sizeof (HLRC_myname));
  strcpy (HLRC_myname, HLRC_myname_short);
  HLRC_HostFullName (HLRC_myname);
  HLRC_HostShortName (HLRC_myname_short);


  /*
    Read parameter values from the environment. If environment
    variable is not defined, use default values specified in
    the source file.
  */
  hosts_file = getenv ("HOSTS_FILE");
  if (hosts_file == NULL)
    {
      hosts_file = getenv ("HOME");
      strcat (hosts_file,"/");
      strcat (hosts_file, HOSTS_FILE);
    }

#ifdef PROT_HANDLE
  mmap_backing_file = getenv ("MMAP_BACKING_FILE");
  if (mmap_backing_file == NULL)
    {
      mmap_backing_file = MMAP_BACKING_FILE;
    }
#endif

  log_file = getenv ("LOG_FILE");
  if (log_file == NULL)
    {
      log_file = LOG_FILE;
    }

  sock_barr_port_str = getenv ("SOCK_BARR_PORT");
  if (sock_barr_port_str == NULL)
    {
      sock_barr_port = SOCK_BARR_PORT;
    }
  else
    {
      sock_barr_port = atoi (sock_barr_port_str);
    }

  /*
    Get the system page size, number of words in
    a page, number of bits to represent pagesize
  */
  pagesize = getpagesize ();
  logpagesize = getlogpagesize (pagesize);
  diff_buff_size = pagesize * 2;


#ifdef PROC_BIND_SUPPORT
  fprintf (logfile, "Coming inside PROC_BIND_SUPPORT\n");
  num_cpu = sysmp (MP_NPROCS);
  if (num_cpu < 0)
    {
      perror ("Getting the number of processors\n");
      HLRC_ErrPrint ("Error in getting number of processors\n");
      exit (1);
    }

  curr_cpu = sysmp (MP_CURPROC);
  if (curr_cpu < 0)
    {
      perror ("Getting the current processor id\n");
      HLRC_ErrPrint ("Error in getting the current processor id\n");
      exit (1);
    }
  fprintf (logfile, "Current CPU: %d[of %d procs]\n", curr_cpu, num_cpu);
  fflush (logfile);

  if (sysmp (MP_MUSTRUN, curr_cpu) < 0)
    {
      perror ("Binding to the current processor\n");
      HLRC_ErrPrint ("Binding to the current processor(%d)\n", curr_cpu);
      exit (1);
    }
  app_cpu = curr_cpu;
  curr_cpu = (curr_cpu + 1) % num_cpu;
#endif



  /*
    Define the indices at which various fields are
    present, in the control message.
  */
  msg_type_index = HLRC_MSG_HDR_TOP;
  msg_node_index = HLRC_MSG_HDR_TOP - 1;
  msg_item_index = HLRC_MSG_HDR_TOP - 2;
  msg_tmst1_start = HLRC_MSG_HDR_TOP - 2 - NPROCS;
  msg_tmst2_start = HLRC_MSG_HDR_TOP - 2 - 2 * NPROCS;

  GETTIME (HLRC_begin);

  /*
    Get the absolute pathname for the stat file
  */
  strcpy (HLRC_stat.abspathstatfile, argv[0]);
  strncat (HLRC_stat.abspathstatfile, STAT_FILE_EXT, 1024);

  strcpy (HLRC_program, argv[0]);

#if 0
  if ((argv[0][0] == '/') && (argv[0][1] == '/'))
    {
      first = 0;
    }
  else
    {
      first = 1;
      /*
	Add a '/' to the pathname for the slave nodes,
	so that the slave nodes can identify themselves
	by the // in the beginning of the pathname.
      */
      HLRC_AbsoluteCurrPath (argv[0], abspathname);
    }
#endif

  HLRC_AbsoluteCurrPath (argv[0], abspathname);

  HLRC_Hosts (hosts_file, &HLRC_nhosts);

  HLRC_nprocs = HLRC_nhosts;
  //  if (HLRC_HostNumber(HLRC_hostnames[serverNodes]) == HLRC_HostNumber(HLRC_myname_short))
  if(!strcmp(HLRC_myname_short, HLRC_hostnames[0])) {

    //fprintf (logfile, "\n\nUsing %s as the hosts file\n", hosts_file);
    //fprintf (logfile, "Running application on %d machines\n", HLRC_nhosts);
    //fprintf (logfile, "Using %s as the backing file for shared memory\n", mmap_backing_file);
    //fprintf (logfile, "Using %s as the log file\n", log_file);
    //fprintf (logfile, "Statistics will be written to %s\n\n", HLRC_stat.abspathstatfile);

    for (i = 1; i < HLRC_nprocs; i++)
      {
	char command_string[512];
	char cmd_string[512];

	bzero (cmd_string, 512);
	bzero (command_string, 512);
	for (j = 1; j < argc; j++)
	  {
	    strcat (command_string, " ");
	    strcat (command_string, argv[j]);
	  }
	sprintf (cmd_string, "ssh %s %s %s\n", HLRC_hostnames[i], abspathname, command_string);	/*lobosco */
	//	  sprintf (cmd_string, "rsh %s %s\n", HLRC_hostnames[i], abspathname, command_string);	/*lobosco */

	/* 
	   Spawn off processes on the other nodes 
	*/
	if (fork () == 0)
	  {
	    system (cmd_string);
	    exit (0);
	  }
      }
  }
  else
    {
      /* Dummy loop for the slave nodes */

      for (i = 1; i < HLRC_nprocs; i++)
	{
	  char command_string[512];
	  char cmd_string[512];

	  bzero (cmd_string, 512);
	  bzero (command_string, 512);

	  /* 
	     Dummy child processes which do nothing
	  */
	  if (fork () == 0)
	    {
	      system (cmd_string);
	      exit (0);
	    }
	}
    }


  // trecho copiado de VIA_EstablishConnections

  /* 
     Determine my own id.  I act as server for all other nodes coming 
     after me in the command line 
  */

  while(strcmp(HLRC_hostnames[serverNodes], HLRC_myname_short) &&
	(serverNodes < HLRC_nprocs)){ 
    /*  while ((HLRC_HostNumber(HLRC_hostnames[serverNodes]) != HLRC_HostNumber(HLRC_myname_short)) &&
	(serverNodes < HLRC_nprocs)) */
    serverNodes++;
  }

  strcpy(HLRC_myname_short,HLRC_hostnames[serverNodes]);
  strcpy(HLRC_myname, HLRC_hostnames[serverNodes]);

  if (serverNodes == HLRC_nprocs)
    {
      HLRC_ErrPrint ("Unable to find Node %s in input file\n", HLRC_myname);
      exit (1);
    }

  HLRC_myself = serverNodes;


  for (i = 0; i < HLRC_nprocs; i++)
    {
      if (TRANS_RegisterGeneralBuffers (i) == TRANS_ERROR)
	{
	  HLRC_ErrPrint ("Unable to register general buffers\n");
	  exit (1);
	}
    }



  CreateBarrierSocket ();


  // Inicializacao do TCP/UDP

  if (TRANS_Init () != 0)
    {
      HLRC_ErrPrint ("TRANSPORT initialization failed\n");
      exit (1);
    }

  HLRC_nprocs = HLRC_nhosts;

  _per_page = (volatile TPerPage *) _AMALLOC (PAGE_NUM * sizeof (TPerPage),
					      DESCRIPTOR_ALIGNMENT);

  _lock_data =
    (volatile TLockData *) _AMALLOC (sizeof (TLockData) * NLOCKS,
				     DESCRIPTOR_ALIGNMENT);

  _global_stamp =
    (volatile int *) _AMALLOC (sizeof (int) * (2 * NPROCS + MSG_HDR_LEN),
			       DESCRIPTOR_ALIGNMENT);

  for (i = 0; i < HLRC_nprocs; i++)
    {
      _diff_control[i] =
	(volatile TDiffControl *) _AMALLOC (sizeof (TDiffControl) *
					    DIFF_WIN_SIZE,
					    DESCRIPTOR_ALIGNMENT);

      buf[i] = (int *) _AMALLOC (sizeof (int) * 2, DESCRIPTOR_ALIGNMENT);
    }

  diffAck = (int *) _AMALLOC (sizeof (int), DESCRIPTOR_ALIGNMENT);
  *diffAck = 1;


#ifdef COPY_ON_RECV
  HLRC_page_recv_buff = (int *) _AMALLOC (pagesize, pagesize);
#endif



  for (i = 0; i < HLRC_nprocs; i++)
    {
      for (j = 0; j < HLRC_nprocs; j++)
	{
	  if (i != j)
	    {
	      HLRC_scratch_buffer[i][j] =
		(int *) _AMALLOC (pagesize, pagesize);
	    }
	}

      _bin[i] = (volatile int *) _AMALLOC (BINSIZE * sizeof (int), pagesize);

      HLRC_request_buffer[i] = (int *) _AMALLOC (pagesize, pagesize);
    }

  _bar_buff = (volatile TBarrierBuff *) _AMALLOC (pagesize, pagesize);

  hlrc_protocol_control =
    (volatile TProtocolControl *) _AMALLOC (sizeof (TProtocolControl),
					    pagesize);

  for (i = 0; i < HLRC_nprocs; i++)
    {
      for (j = 0; j < DIFF_WIN_SIZE; j++)
	{
	  hlrc_protocol_control->diffBufferFree[i][j] = 1;
	}

      _next_diff[i] = 0;
    }


#ifdef PROT_HANDLE
  /*
    Create and open the mmap file to be used as backing store
    for the two handles to shared memory
  */
  mmap_fd = open (mmap_backing_file, O_RDWR | O_CREAT | O_TRUNC, S_IRWXU);
  if (mmap_fd < 0)
    {
      fprintf (logfile, "mmap: file open error\n");
      fflush (logfile);
    }
#ifdef HLRC_DEBUG

  HLRC_ErrPrint ("Creating mmap backing file...\n");

#endif
  /*
    Fill mmap file with as many bytes as the size of shared
    memeory
  */
#if 0
  bzero (zerobuf, pagesize);
  for (i = 0; i < PAGE_NUM; i++)
    {
      if (write (mmap_fd, zerobuf, pagesize) < pagesize)
	{
	  perror ("Creating mmap backing file");
	  fprintf (logfile, "Error creating mmap backing file\n");
	  fflush (logfile);
	  exit (1);
	}
    }
#endif

  lseek (mmap_fd, PAGE_NUM * pagesize, SEEK_SET);
  write (mmap_fd, "\0", 1);
#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Done\n");
#endif
  remappointer = old_first_alloc = _first_alloc = _last_alloc =
    mmap ((void *)0x95000000, pagesize * PAGE_NUM, PROT_READ | PROT_WRITE, 
	  MAP_SHARED|MAP_FIXED, mmap_fd, 0);
#else
  remappointer = _first_alloc = _last_alloc = (int *) malloc (pagesize * PAGE_NUM);
#endif

  if ((_first_alloc) == NULL)
    {
      fprintf (logfile, "allocation error\n");
      fflush (logfile);
      exit (1);
    }
  if (((unsigned) _first_alloc) & 0xfff)
    {
      _first_alloc += pagesize / WORDSIZE;
      //      (unsigned) _first_alloc = (unsigned) _first_alloc & ~0xfff;  Anderson
      _first_alloc = (int *)((unsigned) _first_alloc & ~0xfff);
    }

  start_addr = (int *) _AMALLOC (sizeof (int), DESCRIPTOR_ALIGNMENT);

  *start_addr = (int) _first_alloc;

  for (i = 0; i < HLRC_nprocs; i++)
    {
      pRemoteStartAddr[i] =
	(int *) _AMALLOC (sizeof (int), DESCRIPTOR_ALIGNMENT);
    }

  for (i = 0; i < HLRC_nprocs; i++)
    {
      _diff_buff_send[i] = (int *) _AMALLOC (diff_buff_size * DIFF_WIN_SIZE,
					     pagesize);
      _diff_buff_recv[i] = (int *) _AMALLOC (diff_buff_size * DIFF_WIN_SIZE,
					     pagesize);

      if (_diff_buff_send[i] == NULL || _diff_buff_recv[i] == NULL)
	{
	  HLRC_ErrPrint ("Failed to allocate diff buffers\n");
	  exit (1);
	}

      pDiffBuffs[i] =
	(TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
				    DESCRIPTOR_ALIGNMENT);
    }


  _bar_end_buff = (int *) _AMALLOC (pagesize, pagesize);

  for (i = 0; i < HLRC_nprocs; i++)
    _bar_end_buff[i] = -1;

  prev_barr_complete = 1;

#ifdef COPY_ON_SEND
  for (i = 0; i < HLRC_nprocs; i++)
    {
      HLRC_page_send_buff[i] = (int *) _AMALLOC (pagesize, pagesize);
    }
#else
  for (i = 0; i < HLRC_nprocs; i++)
    {
      for (j = 0; j < MAX_REG_PART; j++)
	{
	  pRemoteMemory[i][j] =
	    (TRemoteMemory *) _AMALLOC (sizeof (TRemoteMemory),
					DESCRIPTOR_ALIGNMENT);
	}
    }
#endif


#ifdef PROT_HANDLE
  /*
    Allocate memory for the protocol handle to shared memory
  */
  prot_first_alloc =
    mmap (NULL, pagesize * PAGE_NUM, PROT_READ | PROT_WRITE, MAP_SHARED,
	  mmap_fd, 0);
  if ((prot_first_alloc) == MAP_FAILED)
    {
      fprintf (logfile, "mmap: allocation error\n");
      fflush (logfile);
      exit (1);
    }
  num_reg_part = (PAGE_NUM + MAX_REGMEM_PAGES - 1) / MAX_REGMEM_PAGES;
#endif

  for (i = 0; i < HLRC_nprocs; i++)
    {
      _index_bin[i] = 0;
    }

  for (i = 0; i < NLOCKS; i++)
    {
      if ((i % HLRC_nprocs) == HLRC_myself)
	{
	  _lock_data[i].state = 1;
	  _lock_data[i].last = HLRC_myself;
	}
    }

  logfile = fopen (log_file, "w");	//lobosco:"a"

  fprintf (logfile, "START\n");

  fflush (logfile);
  fprintf (logfile, "MYSELF %d\n", HLRC_myself);

  fflush (logfile);


  /*
    Exchange the starting addresses of the shared memory
    regions on different nodes
  */
  SocketBarrier ();

  if (TRANS_ExchangeDMAMappings (START_ADDR_MESG, 0) != TRANS_SUCCESS)
    {
      HLRC_ErrPrint ("Unable to exchange start addresses\n");
      exit (1);
    }


  /*
    Synchronize the virtual address spaces of the shared memory
    regions on all nodes
  */
  for (i = 0; i < HLRC_nprocs; i++)
    {
      if (i == HLRC_myself)
	continue;
      //fprintf (logfile, "Synchronize: i=%d _first_alloc=%x _last_alloc=%x *pRemoteStartAddr[i]=%x last addr=%x\n", i, _first_alloc, _last_alloc ,*pRemoteStartAddr[i], (_first_alloc + PAGE_NUM*pagesize));
      if ((unsigned) _first_alloc < (unsigned)*pRemoteStartAddr[i])
	{
	  //	  if (*pRemoteStartAddr[i] > (_first_alloc + PAGE_NUM*pagesize)) { Anderson
	  if ((unsigned)(*pRemoteStartAddr[i]) > (unsigned)(_first_alloc + PAGE_NUM*pagesize)) { 

	    //fprintf(logfile,"Remapmemory 1: a=%x> b=%x\n",*pRemoteStartAddr[i],(_first_alloc + PAGE_NUM*pagesize));fflush(logfile);
	    remapmemory = 1;
	  }
	  _first_alloc = (int *) (*pRemoteStartAddr[i]);
	}
      if ((unsigned) _last_alloc > (unsigned)*pRemoteStartAddr[i])
	{
	  //if ((*pRemoteStartAddr[i] + pagesize*PAGE_NUM) <  _last_alloc) { Anderson
	  if ((unsigned)(*pRemoteStartAddr[i] + pagesize*PAGE_NUM) <  (unsigned)(_last_alloc)) {

	    //fprintf(logfile,"Remapmemory 2: a=%x < b=%x",(*pRemoteStartAddr[i] + pagesize*PAGE_NUM),_last_alloc);fflush(logfile);
	    remapmemory = 1;
	  }
	  _last_alloc = (int *) (*pRemoteStartAddr[i]);
	}
    }

  _size_adjust = ((unsigned) (_first_alloc - _last_alloc)) >> logpagesize;

  //fprintf(logfile,"Adjust = %x",_size_adjust);fflush(logfile);

  if (remapmemory) {
    munmap (remappointer,  pagesize * PAGE_NUM);
    old_first_alloc = _first_alloc = _last_alloc =
      mmap (_first_alloc, pagesize * PAGE_NUM, PROT_READ | PROT_WRITE, MAP_SHARED | MAP_FIXED, mmap_fd, 0);
    if ((_first_alloc)==MAP_FAILED) {
      fprintf (logfile, "mmap: allocation error\n");
      fflush (logfile);
      exit (1);
    }
    fprintf (logfile, "_first_alloc = %x\n", (unsigned int) _first_alloc);
    fflush (logfile);
    _size_adjust = ((unsigned) (_first_alloc - _last_alloc)) >> logpagesize;
  }

  fprintf (logfile, "SBRK %x\n", (unsigned int)_first_alloc);
  fflush (logfile);

#ifdef PROT_HANDLE
  /*
    Close and remove the backing file from the hard disk
  */

  close (mmap_fd);
  unlink (mmap_backing_file);

  prot_first_alloc += (int) (_first_alloc - old_first_alloc);

  fprintf (logfile, "prot_first_alloc = %x\n", (int) prot_first_alloc);
  fflush (logfile);

  {
    int *addr = prot_first_alloc;
    int *prot_last_alloc = //prot_first_alloc +(PAGE_NUM*pagesize)/sizeof(int);
      (prot_first_alloc - ((unsigned) (_first_alloc - _last_alloc))) +
      ((PAGE_NUM * pagesize) / sizeof (int));	//lobosco
    int regsize;
    int num_part = 0;

    prot_first_shared_page = (int) prot_first_alloc >> logpagesize;

    while (addr < prot_last_alloc)
      {

	if ((regsize =
	     (((int) prot_last_alloc - (int) addr))) >=
	    pagesize * MAX_REGMEM_PAGES)
	  {
	    regsize = pagesize * MAX_REGMEM_PAGES;
	  }

	num_part++;
	addr += regsize / sizeof (int);
      }
  }

  for (i = 0; i < num_reg_part; i++)
    {
      if (TRANS_ExchangeDMAMappings (SVM_ADDR_MESG, i) != TRANS_SUCCESS)
	{
	  HLRC_ErrPrint ("Unable to exchange start addresses\n");
	  exit (1);
	}
    }

#endif

#ifdef COPY_ON_RECV
#ifdef HLRC_DEBUG
  //  HLRC_ErrPrint ("Memory Handle: %x\n", hPageRecvBuffHandle);
#endif

  if (TRANS_ExchangeDMAMappings (PAGE_MESG, 0) != TRANS_SUCCESS)
    {
      HLRC_ErrPrint ("Unable to exchange DMA mappings(PAGE_MESG)\n");
      exit (1);
    }
#endif

#ifdef HLRC_DEBUG
  //  HLRC_ErrPrint ("Memory Handle: %x\n", hProtocolControlHandle);
#endif

  if (TRANS_ExchangeDMAMappings (DIFFACK_MESG, 0) != TRANS_SUCCESS)

    {
      HLRC_ErrPrint ("Unable to exchange DMA mappings(DIFFACK_MESG)\n");
      exit (1);
    }

#ifdef HLRC_DEBUG
  //  HLRC_ErrPrint ("Memory Handle: %x\n", hBarrierBufferHandle);
#endif

  if (TRANS_ExchangeDMAMappings (BARRIER_MESG, 0) != TRANS_SUCCESS){
    HLRC_ErrPrint ("Unable to exchange DMA mappings(BARRIER_MESG)\n");
    exit (1);
  }

  for (i = 0; i < HLRC_nprocs; i++)
    {

#ifdef HLRC_DEBUG
      //      HLRC_ErrPrint ("Memory Handle(req, %d): %x\n", i,
      //		     hLocalRequestBufferHandles[i]);
#endif

      if (TRANS_ExchangeDMAMappings (REQUEST_MESG, i) != TRANS_SUCCESS)
	{
	  HLRC_ErrPrint ("Unable to exchange DMA mappings(REQUEST_MESG)\n");
	  exit (1);
	}
    }

  for (i = 0; i < HLRC_nprocs; i++)
    {

#ifdef HLRC_DEBUG
      //      HLRC_ErrPrint ("Memory Handle: %x(bin, %d)\n", hBinHandles[i], i);
#endif

      if (TRANS_ExchangeDMAMappings (BIN_MESG, i) != TRANS_SUCCESS)
	{
	  HLRC_ErrPrint ("Unable to exchange DMA mappings(BIN_MESG)\n");
	  exit (1);
	}
    }

  if (TRANS_ExchangeDMAMappings (DIFF_MESG, 0) != TRANS_SUCCESS)
    {
      HLRC_ErrPrint ("Unable to exchange DMA mappings(DIFF_MESG)\n");
      exit (1);
    }

#ifdef HLRC_DEBUG
  //  HLRC_ErrPrint ("Memory Handle: %x\n", hBarrEndBuffHandle);
#endif

  if (TRANS_ExchangeDMAMappings (BARR_END_MESG, 0) != TRANS_SUCCESS)
    {
      HLRC_ErrPrint ("Unable to exchange DMA mappings(BARR_END_MESG)\n");
      exit (1);
    }



#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Phase 1 completed\n");
  fflush (stderr);
#endif
  gettimeofday (&HLRC_internal_time, NULL);
  HLRC_start = HLRC_internal_time.tv_sec * 1000 +
    HLRC_internal_time.tv_usec / 1000;

  for (i = 0; i < BARR_WIN_SIZE; i++)
    {
      for (j = 0; j < HLRC_nprocs; j++)
	{
	  _bar_buff[i].buf[j][1] = -1;
	}
    }


}

/*
  HLRC_InitParallel: Initialization for the parallel phase
*/

void
HLRC_InitParallel ()
{
  register int i, j, k;
  int error;
  int size;
  pthread_attr_t attr, attr_rdma;
  int nodeId;
  int *twins_mem;
  int viHandle;

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("INIT PARALLEL: %d => shared data %d pages \n",
		 HLRC_myself, HLRC_total_pages);
  fflush (stderr);
  fprintf (logfile, "INIT PARALLEL: %d => shared data %d pages \n", HLRC_myself, HLRC_total_pages);
  fflush (logfile);
#endif

  if (HLRC_init_flag)
    return;

  for (i = 0; i < HLRC_nprocs; i++)
    HLRC_stats_buff[i] = HLRC_Malloc (pagesize);

  _nopage = _last_shared_page - _first_shared_page + 1;

  twins_mem =
    (int *) _AMALLOC ((MAX_TWINS / HLRC_nprocs) * pagesize, pagesize);

  for (i = 0; i < (MAX_TWINS / HLRC_nprocs); i++)
    //    (int *) _AMALLOC ((NUPDATES / HLRC_nprocs) * pagesize, pagesize);
    //  for (i = 0; i < (NUPDATES / HLRC_nprocs); i++)
    {
      _free_twins[_notwins++] = twins_mem;
      twins_mem += pagesize / sizeof (int);
    }

  /*
    Prepare the diff_control structure to be used
    for sending DIFFs.
  */
  for (i = 0; i < HLRC_nprocs; i++)
    {
      for (k = 0; k < DIFF_WIN_SIZE; k++)
	{
	  _diff_control[i][k].no = k;
	  _diff_control[i][k].proc = HLRC_myself;
	  _diff_control[i][k].cmd = DIFF;
	}
    }

  for (i = 0; i < HLRC_nprocs; i++)
    {
      _home_flush[i] =
	(volatile int *) _AMALLOC (_nopage * sizeof (unsigned), pagesize);
    }


  for (i = 0; i < HLRC_nlocks; i++)
    {
      if ((i % HLRC_nprocs) == HLRC_myself)
	{
	  _lock_data[i].state = 1;
	  _lock_data[i].last = HLRC_myself;
	}
    }

  /*
    Prepare the global_stamp structure to be used
    for sending LOCK requests. Since the global_stamp
    represents the global time, it is used in making
    requests for locks. The global_stamp vector is
    extended to accomodate the control information
    sent along with a LOCK request. 
  */
  for (i = 0; i < NPROCS; i++)
    _global_stamp[i] = 0;

  _global_stamp[NPROCS] = 0;
  _global_stamp[NPROCS + 1] = HLRC_myself;
  _global_stamp[NPROCS + 2] = LOCK;

  /*
    Set up the handler for page faults
  */
  //  HLRC_sig.sa_handler = (void (*)(int)) HLRC_PageHandler; //original
  // HLRC_sig.sa_sigaction = (void (*)(int)) HLRC_PageHandler; Anderson
  HLRC_sig.sa_sigaction = (void (*)(int, siginfo_t *, void *)) HLRC_PageHandler;
  // HLRC_sig.sa_flags = 0; //original
  HLRC_sig.sa_flags = SA_SIGINFO;
  sigaction (SIGSEGV, &HLRC_sig, 0);


  //  HLRC_sig.sa_handler = (void (*)(int)) HLRC_PageHandler; //original
  //  HLRC_sig5.sa_sigaction = (void (*)(int)) HLRC_PageHandler5; Anderson
 HLRC_sig5.sa_sigaction = (void (*)(int, siginfo_t *, void *)) HLRC_PageHandler5;

  // HLRC_sig.sa_flags = 0; //original
  //HLRC_sig5.sa_flags = SA_SIGINFO;
  sigaction (SIGTRAP, &HLRC_sig5, 0);

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Phase 2 completed\n");
#endif

  for (i = _first_shared_page; i <= _last_shared_page; i++)
    {

      /*
	The my_stamp vector, which represents the version of
	the page, is included in PAGE requests. Here, the
	my_stamp vector is initialized with the control info
	sent along with the request for the page.
      */
      _per_page[i - _first_shared_page].my_stamp[NPROCS] =
	i - _first_shared_page;
      _per_page[i - _first_shared_page].my_stamp[NPROCS + 1] = HLRC_myself;
      _per_page[i - _first_shared_page].my_stamp[NPROCS + 2] = PAGE;

      /*
	Protect all shared memory pages to trap access
      */
      if (((i - _first_shared_page) % HLRC_nprocs) == HLRC_myself)
	{
	  _per_page[i - _first_shared_page].state = 1;
	  mprotect ((void *) (i << logpagesize), pagesize, HLRC_PROT_NONE);
	}
      else
	{
	  _per_page[i - _first_shared_page].state = 0;
	  mprotect ((void *) (i << logpagesize), pagesize, HLRC_PROT_NONE);
	}
#ifdef INDIRECT_DIFFS
      _per_page[i - _first_shared_page].dirty_and_invalid = 0;
#endif
    }

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Phase 3 completed\n");
#endif

  /*
    Initialize mutexes used for critical section
  */
  pthread_mutex_init (&HLRC_critical_section_mutex[0], NULL);
  pthread_mutex_init (&HLRC_critical_section_mutex[1], NULL);

  /*
    Now, create a new thread to handle all incoming requests
  */
  pthread_attr_init (&attr);
  pthread_attr_setdetachstate (&attr, PTHREAD_CREATE_JOINABLE);

  if (pthread_create (&hServerThreadHandle, &attr, HLRC_HandleRequests, NULL)){
    HLRC_ErrPrint ("Unable to create thread to service requests=> %i\n", errno);
    exit (1);
  }
  /*
    pthread_attr_init(&attr_rdma);
    pthread_attr_setdetachstate(&attr_rdma, PTHREAD_CREATE_JOINABLE);

    if (pthread_create(&hServerThreadRDMAHandle, &attr_rdma, HLRC_HandleRDMARequests, NULL)){
    HLRC_ErrPrint("Unable to create thread to service requests\n");
    exit(1);
    }
  */
  app_thread = pthread_self ();
  app_pid = getpid ();
  srv_thread = hServerThreadHandle;
  //  srv_rdma_thread = hServerThreadRDMAHandle;

#ifdef HLRC_DEBUG
  fprintf (logfile, "Thread identifier: %i\n", (int)hServerThreadHandle);
  fflush (logfile);
#endif
  SocketBarrier ();
#ifdef HLRC_DEBUG
  fprintf (logfile, "SockBarrierCompl\n");
  fflush (logfile);
#endif

  HLRC_init_flag = 1;
}


/*
  HLRC_Reset:
  Reset all the fields of the statistics structure
*/
void
HLRC_Reset ()
{
  struct rusage ru;

  HLRC_reset_flag = 1;
  HLRC_stat.time_exec = 0;
  HLRC_stat.time_comm = 0;
  HLRC_stat.time_lock = 0;
  HLRC_stat.time_barr = 0;
  HLRC_stat.time_over = 0;
  HLRC_stat.time_handler = 0;
  HLRC_stat.time_comp = 0;

  HLRC_stat.page_faults = 0;
  HLRC_stat.page_misses = 0;
  HLRC_stat.diff_created = 0;
  HLRC_stat.diff_applied = 0;
  HLRC_stat.forward = 0;

  HLRC_stat.lock_acq = 0;
  HLRC_stat.lock_remote = 0;
  HLRC_stat.barriers = 0;
  HLRC_stat.msg_control = 0;
  HLRC_stat.msg_data = 0;
  HLRC_stat.traffic_control = 0;
  HLRC_stat.traffic_barr = 0;
  HLRC_stat.traffic_data = 0;
  HLRC_stat.traffic_diff = 0;
  HLRC_stat.requests_recv = 0;

  getrusage (RUSAGE_SELF, &ru);
  HLRC_stat.num_swaps = ru.ru_nswap;

  HLRC_stat.diffwait = 0;
  HLRC_stat.time_spinning = 0;

  gettimeofday (&HLRC_internal_time, NULL);
  GETTIME (HLRC_begin);

#ifdef HLRC_DEBUG
  HLRC_ErrPrint ("Reset start time\n");
  fprintf (logfile, "Reset start time\n");
  fflush (logfile);
#endif

  HLRC_start = HLRC_internal_time.tv_sec * 1000 +
    HLRC_internal_time.tv_usec / 1000;

}


/*
  HLRC_ClockStop:
  Stop the timing, collect the statistics into a buffer
  and print them in the statistics file
*/
void
HLRC_ClockStop ()
{
  int i;
  FILE *fp;
  struct rusage ru;


  HLRC_clock_stopped = 1;

  GETTIME (HLRC_final);

  fprintf (logfile, "Time from cycle counter: %d\n",
	   (unsigned) ((HLRC_final - HLRC_begin) / CPU_MHZ));
  fflush (logfile);

  gettimeofday (&HLRC_internal_time, NULL);
  HLRC_stat.time_exec = HLRC_internal_time.tv_sec * 1000 +
    HLRC_internal_time.tv_usec / 1000 - HLRC_start;

  getrusage (RUSAGE_SELF, &ru);
  HLRC_stat.num_swaps = (ru.ru_nswap - HLRC_stat.num_swaps);

  HLRC_stats_buff[HLRC_myself][0] = 0;

  HLRC_Barrier ();

  for (i = 0; i < HLRC_nprocs; i++)
    {
      char tmp;
      tmp = HLRC_stats_buff[i][0];
    }

  HLRC_Barrier ();

  HLRC_PrintStatsInBuffer ();

  HLRC_Barrier ();

  if (HLRC_myself == 0)
    {
      fp = fopen (HLRC_stat.abspathstatfile, "w");

      for (i = 0; i < HLRC_nprocs; i++)
	{
	  fprintf (fp, "%s\n\n\n", HLRC_stats_buff[i]);
	}
      fclose (fp);
    }
}


/*
  HLRC_End: Release all the resources used and exit
*/
void
HLRC_End ()
{
  unsigned i;

  fprintf (logfile, "HLRC END %d\n", HLRC_myself);
  fflush (logfile);
  //HLRC_ErrPrint ("HLRC END %d\n", HLRC_myself);
  //fflush (stderr);

  HLRC_Barrier ();

  HLRC_init_flag = 0;

  /*  pthread_kill (hServerThreadHandle, SIGTERM);

  pthread_join (hServerThreadHandle, NULL);*/

  HLRC_ReleaseResources ();

  exit (0);
}


/*
  HLRC_Acquire: Acquire the specified lock. 
  The locks are managed using a distributed protocol in which
  each lock has a home, which is statically assigned. A modulo 
  scheme is used on the lock id to determine the home of the
  lock. All nodes except the home of the lock, send their lock
  requests to the home. The home remembers the last requester
  of the lock and forwards the request to the last requester
  and updates its last requester info. With this scheme, 
  we have a distributed queue for the lock reuests, with a max
  of one request in any node's queue.
*/

void
HLRC_Acquire (int lock)
{
  unsigned long long time1, time2;

#ifdef HLRC_DEBUG
  fprintf (logfile, "HLRC_Acquire: Trying to acquire lock %d\n", lock);
  fflush (logfile);
#endif

  GETTIME (time1);
  HLRC_stat.lock_acq++;

  HLRC_EnterCriticalSection (0);
  /* 
     If lock is available locally, use it
  */

 lockagain:

  if (_lock_data[lock].state)
    {
      _lock_data[lock].state = 0;
      _lock_data[lock].last = HLRC_myself;
      HLRC_LeaveCriticalSection (0);
      GETTIME (time2);
      if (!HLRC_clock_stopped)
	HLRC_stat.time_lock += time2 - time1;

#ifdef HLRC_DEBUG
      fprintf (logfile, "HLRC_Acquire returning... : Lock %d acquired\n", lock);
      fflush (logfile);
#endif
      return;
    }

  /*
    Fetch lock from remote node
  */

  if ((lock % HLRC_nprocs) == HLRC_myself)
    {

      if (_lock_data[lock].last != HLRC_myself) {

	/*
	  At the home of the lock:
	  Send the request to the last requester
	*/

	HLRC_stat.lock_remote++;

	_global_stamp[NPROCS] = lock;
	HLRC_SendRequest (_lock_data[lock].last, HLRC_myself,
			  (msg_tmst1_start) * WORDSIZE, _global_stamp,
			  (NPROCS + MSG_HDR_LEN) * sizeof (int), LOCK);
	_lock_data[lock].last = HLRC_myself;

	HLRC_LeaveCriticalSection (0);
      }
      else {
	HLRC_LeaveCriticalSection(0);
	pthread_mutex_lock(&lmutex_);
	_lock_data[lock].state = 3;
	pthread_cond_wait(&lcond_, &lmutex_);
	pthread_mutex_unlock(&lmutex_);

	HLRC_EnterCriticalSection(0);
	goto lockagain;
      }
    }
  else
    {

      /*
	Send the request to the home of the lock 
      */

      HLRC_stat.lock_remote++;

      _global_stamp[NPROCS] = lock;
      HLRC_SendRequest (lock % HLRC_nprocs, HLRC_myself,
			(msg_tmst1_start) * WORDSIZE, _global_stamp,
			(NPROCS + MSG_HDR_LEN) * sizeof (int), LOCK);

      HLRC_LeaveCriticalSection (0);
    }

  /*
    Start the interval clean by closing the current interval
  */

  _end_interval ();

  /* 
     Receive lock and write notices 
  */

  HLRC_Recv (LOCK, 0, 0, lock);

  GETTIME (time2) if (!HLRC_clock_stopped)
    HLRC_stat.time_lock += time2 - time1;
#ifdef HLRC_DEBUG
  fprintf (logfile, "Returning... HLRC_Acquire: Lock %d acquired\n", lock);
  fflush (logfile);
#endif

}


/*
  HLRC_Release: Release the specified lock
*/

void
HLRC_Release (int lock)
{
  register int i;
  unsigned long long time1, time2;

#ifdef HLRC_DEBUG
  fprintf (logfile, "HLRC_Release: %d\n", lock);
  fflush (logfile);
#endif

  GETTIME (time1);

  /* 
     Close the interval
  */
  _end_interval ();

  for (i = 0; i < HLRC_nprocs; i++)
    {
      _lock_data[lock].stamp[i] = _global_stamp[i];
      _lock_data[lock].index[i] = _index_bin[i];
    }

  HLRC_EnterCriticalSection (0);
  if (_lock_data[lock].state == 0 || _lock_data[lock].state == 3)
    {
      _lock_data[lock].state = 1;
      pthread_cond_signal(&lcond_);
      HLRC_LeaveCriticalSection (0);
    }
  else
    {

      /*
	If there is a pending request for this lock
	service the request.
      */

      HLRC_LeaveCriticalSection (0);
      for (i = 0; i < HLRC_nprocs; i++)
	{
	  if (i != HLRC_myself)
	    {
	      if ((_pending[i].type == LOCK) && (_pending[i].addr == lock))
		{
		  _pending[i].type = 0;
		  _send_lock (APP_THREAD, lock, i, _pending[i].stamp);
		  break;
		}
	    }
	}
    }

  GETTIME (time2);
  if (!HLRC_clock_stopped)
    {
      HLRC_stat.time_over += (time2 - time1);
    }

}

void
HLRC_RecAcquire(int lock)
{

  HLRC_EnterCriticalSection(0);
  if (_lock_data[lock].counter == 0 )
    {
      HLRC_LeaveCriticalSection(0);
      HLRC_Acquire(lock);

      HLRC_EnterCriticalSection(0);
      _lock_data[lock].counter = 1;
      HLRC_LeaveCriticalSection(0);
    }
  else
    {
      if (_lock_data[lock].owner == HLRC_myself) {
	_lock_data[lock].counter++;
	HLRC_LeaveCriticalSection(0);
      }
      else{
	HLRC_LeaveCriticalSection(0);
	HLRC_Acquire(lock);

	HLRC_EnterCriticalSection(0);
	_lock_data[lock].counter = 1;
	HLRC_LeaveCriticalSection(0);
      }

    }

}

void
HLRC_RecRelease(int lock)
{

  HLRC_EnterCriticalSection(0);
  if (_lock_data[lock].counter )
    {
      if (_lock_data[lock].owner == HLRC_myself) 
	if (--_lock_data[lock].counter == 0){
	  HLRC_LeaveCriticalSection(0);
	  HLRC_Release(lock);
	  return;
	}

    }
  HLRC_LeaveCriticalSection(0);

}

/*
  HLRC_Barrier:
  Establish a synchronization point by broadcasting a
  BARRIER message and waiting for the BARRIER message
  from everyone else. At the barrier, the list of dirty
  pages is cleared by flushing DIFFs to the homes of 
  respective pages, and the write notices generated 
  locally are broadcast to everyone. The _bin data
  structure, which stores the write notices is reset
  by resetting the _index_bin data structure.
*/

void
HLRC_Barrier ()
{
  register int i, j, k, l, index;
  int counter = 1, begin, size;	//lobosco;
  unsigned long long time1, time2;
  int offset;
  GETTIME (time1);

#ifdef HLRC_DEBUG
  fprintf (logfile, "HLRC_Barrier begin: count %d, flag %d\n", _bar_count,
	   _bar_buff[_bar_count % BARR_WIN_SIZE].buf[1 - HLRC_myself][1]);
  fflush (logfile);
#endif

  if (!HLRC_clock_stopped)
    HLRC_stat.barriers++;

  /*
    Close the interval
  */

  _end_interval ();

  /*
    Send the bins to everyone
  */

  for (i = (HLRC_myself + 1) % HLRC_nprocs; i != HLRC_myself;
       i = (i + 1) % HLRC_nprocs)
    {
      if (_global_stamp[HLRC_myself] > _last_stamp[i])
	{
	  for (k = 0, index = _index_bin[HLRC_myself];
	       k < (_global_stamp[HLRC_myself] - _last_stamp[i]); k++)
	    {
	      index = _bin[HLRC_myself][index - 1];
	    }

	  if (!HLRC_clock_stopped)
	    {
	      HLRC_stat.msg_control++;
	      HLRC_stat.traffic_control +=
		(_index_bin[HLRC_myself] - index) * WORDSIZE;
	      HLRC_stat.traffic_barr +=
		(_index_bin[HLRC_myself] - index) * WORDSIZE;
	    }
	  HLRC_EnterCriticalSection (0);

	  if (_global_stamp[HLRC_myself] > _last_stamp[i])
	    {

#ifdef HLRC_DEBUG
	      fprintf (logfile, " Offset = %d, size = %d\n",
		       index * sizeof (int),
		       (_index_bin[HLRC_myself] - index) * sizeof (int));
	      fflush (logfile);

#endif

	      HLRC_SendMessage (i, HLRC_myself,
				index * sizeof (int),
				(char *) &_bin[HLRC_myself][index],
				(_index_bin[HLRC_myself] -
				 index) * sizeof (int), BIN_MESG, 0);

	    }

	  HLRC_LeaveCriticalSection (0);

	}

#ifdef HLRC_DEBUG
      fprintf (logfile, "HLRC_Barrier: Stage 1\n");
      fflush (logfile);
#endif

      /*
	Send a message with control info about the barrier
      */

      buf[i][1] = _last_stamp[i] = _global_stamp[HLRC_myself];
      buf[i][0] = _index_bin[HLRC_myself];

      if (!HLRC_clock_stopped)
	{
	  HLRC_stat.msg_control++;
	  HLRC_stat.traffic_control += BARR_WIN_SIZE * WORDSIZE;
	}

      offset =
	(((long) &(_bar_buff[_bar_count % BARR_WIN_SIZE].buf[HLRC_myself])) -
	 ((long) (_bar_buff)));

#ifdef HLRC_DEBUG
      fprintf (logfile, " Offset = %d, (%i - %i) \n", offset,
	       (int)(&(_bar_buff[_bar_count % BARR_WIN_SIZE].buf[HLRC_myself])),
	       ((int) (_bar_buff)));
      fflush (logfile);

#endif
      HLRC_SendMessage (i, HLRC_myself, offset, (char *) buf[i],
			2 * sizeof (int), BARRIER_MESG, 0);
    }

#ifdef HLRC_DEBUG
  fprintf (logfile, "HLRC_Barrier: Stage 2\n");
  fflush (logfile);

#endif

  while (counter != HLRC_nprocs)
    {
      sem_wait (&sem_barrier);
      sem_wait (&mutex_barrier);

      for (l = in_fila; tam_fila > 0; l = (l + 1) % HLRC_nprocs)
	{
	  i = fila[l];
	  in_fila = (in_fila + 1) % HLRC_nprocs;
	  tam_fila--;
	  if ((i != HLRC_myself)
	      && (_bar_buff[_bar_count % BARR_WIN_SIZE].buf[i][1] >= 0))
	    {
#ifdef HLRC_DEBUG
	      fprintf (logfile, "HLRC_Barrier: Received message from %d\n",
		       i);
	      fflush (logfile);

#endif
	      _apply_wn (_bar_buff[_bar_count % BARR_WIN_SIZE].buf[i][0],
			 _bar_buff[_bar_count % BARR_WIN_SIZE].buf[i][1], i);
	      _bar_buff[_bar_count % BARR_WIN_SIZE].buf[i][1] = -1;

	      counter++;
	    }
	}
      sem_post (&mutex_barrier);
    }


  /* 
     Send a message to signal the end of barrier, to all other nodes. 
  */

  for (i = 0; i < HLRC_nprocs; i++)
    {
      if (i == HLRC_myself)
	continue;

      HLRC_SendMessage (i, HLRC_myself,
			HLRC_myself * sizeof (int), (char *) diffAck,
			sizeof (int), BARR_END_MESG, 0);
    }

  prev_barr_complete = 0;


  for (i = 0; i < HLRC_nprocs; i++)
    {
      _index_bin[i] = 0;
    }
  _bar_count++;


  GETTIME (time2);
  if (!HLRC_clock_stopped)
    {
      HLRC_stat.time_barr += (time2 - time1);
    }

#ifdef HLRC_DEBUG
  fprintf (logfile, "HLRC_Barrier end\n");
  fflush (logfile);
#endif

}

void HLRC_lock_cond_wait(unsigned lock, unsigned cond)
{

  int lmanager = lock % HLRC_nprocs;

#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_lock_cond_wait -> lock:%i cond:%i\n",
	  HLRC_myself, lock, cond);
  fflush(logfile);
#endif

  HLRC_Release(lock);

#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_Release(%i)\n", HLRC_myself, lock);
  fflush(logfile);
#endif

  HLRC_EnterCriticalSection(0);

  if (lmanager == HLRC_myself) {
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] Eu sou o gerente(%i)\n", HLRC_myself, lock);
    fflush(logfile);
#endif

    _lock_data[lock].wait[HLRC_myself] = 1;
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] lock_data[%i].wait[%i] = 1\n", HLRC_myself,
	    lock, HLRC_myself);
    fflush(logfile);
#endif

  } else {

    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_type_index] = WAIT;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_node_index] =
      HLRC_myself;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_item_index] = lock;

    HLRC_SendRequest(lmanager, HLRC_myself,
		     (msg_tmst1_start) * WORDSIZE,
		     &HLRC_scratch_buffer[HLRC_myself][lmanager]
		     [msg_tmst1_start],
		     (NPROCS + MSG_HDR_LEN) * sizeof(int), WAIT);

#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] HLRC_SendRequest (WAIT) dest:%i\n",
	    HLRC_myself, lmanager);
    fflush(logfile);
#endif
  }

  HLRC_LeaveCriticalSection(0);

#ifdef BUSY_WAIT
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] Bloqueado...BUSY_WAIT....\n", HLRC_myself);
  fflush(logfile);
#endif
  HLRC_Recv(SIGNAL, 0, 0, 0);
#else
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] Bloqueado....pthread_cond_wait...\n",
	  HLRC_myself);
  fflush(logfile);
#endif
  pthread_mutex_lock(&mutex_);
  pthread_cond_wait(&cond_, &mutex_);
  pthread_mutex_unlock(&mutex_);
#endif
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] Chegou sinal \n", HLRC_myself);
  fflush(logfile);
#endif
  HLRC_Acquire(lock);
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_Acquire(%i)", HLRC_myself, lock);
  fflush(logfile);
#endif
}

void
HLRC_lock_cond_timed_wait(unsigned lock, unsigned cond,
			  struct timespec timeout)
{

  int lmanager = lock % HLRC_nprocs;
  int timed;

#ifdef HLRC_DEBUG
  fprintf(logfile,
	  "[%i] HLRC_lock_cond_wait -> lock:%i cond:%i time:%i\n",
	  HLRC_myself, lock, cond, (int)timeout.tv_sec);
  fflush(logfile);
#endif

  HLRC_Release(lock);

#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_Release(%i)\n", HLRC_myself, lock);
  fflush(logfile);
#endif

  HLRC_EnterCriticalSection(0);
  if (lmanager == HLRC_myself) {
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] Eu sou o gerente(%i)\n", HLRC_myself, lock);
    fflush(logfile);
#endif

    _lock_data[lock].wait[HLRC_myself] = 1;
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] lock_data[%i].wait[%i] = 1\n", HLRC_myself,
	    lock, HLRC_myself);
    fflush(logfile);
#endif

  } else {

    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_type_index] = WAIT;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_node_index] =
      HLRC_myself;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_item_index] = lock;

    HLRC_SendRequest(lmanager, HLRC_myself,
		     (msg_tmst1_start) * WORDSIZE,
		     &HLRC_scratch_buffer[HLRC_myself][lmanager]
		     [msg_tmst1_start],
		     (NPROCS + MSG_HDR_LEN) * sizeof(int), WAIT);
  }
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] Bloqueado.......\n", HLRC_myself);
  fflush(logfile);
#endif
  HLRC_LeaveCriticalSection(0);

#ifdef BUSY_WAIT
  timed = HLRC_RecvTimeOut(SIGNAL, timeout);
#else
  pthread_mutex_lock(&mutex_);
  timed = pthread_cond_timedwait(&cond_, &mutex_, &timeout);
  pthread_mutex_unlock(&mutex_);
#endif

#ifdef BUSY_WAIT
  if (timed)
#else
    if (timed == ETIMEDOUT)
#endif
      HLRC_EnterCriticalSection(0);
  if (lmanager == HLRC_myself) {
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] Eu sou o gerente(%i)\n", HLRC_myself, lock);
    fflush(logfile);
#endif

    _lock_data[lock].wait[HLRC_myself] = 0;
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] lock_data[%i].wait[%i] = 1\n", HLRC_myself,
	    lock, HLRC_myself);
    fflush(logfile);
#endif

  } else {
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_type_index] =
      TIMEDOUT;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_node_index] =
      HLRC_myself;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_item_index] = lock;

    HLRC_SendRequest(lmanager, HLRC_myself,
		     (msg_tmst1_start) * WORDSIZE,
		     &HLRC_scratch_buffer[HLRC_myself][lmanager]
		     [msg_tmst1_start],
		     (NPROCS + MSG_HDR_LEN) * sizeof(int), TIMEDOUT);
  }
  HLRC_LeaveCriticalSection(0);
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] Chegou sinal \n", HLRC_myself);
  fflush(logfile);
#endif
  HLRC_Acquire(lock);
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_Acquire(%i)", HLRC_myself, lock);
  fflush(logfile);
#endif
}

void HLRC_lock_cond_signal(unsigned lock, unsigned cond)
{
  int lmanager = lock % HLRC_nprocs;
  int i;
#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_lock_cond_signal lock:%i\n", HLRC_myself,
	  lock); fflush(logfile);
#endif
  /* Eu sou o gerente */
  if (lmanager == HLRC_myself) {

#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] Eu sou o gerente\n", HLRC_myself);
    fflush(logfile);
#endif

    HLRC_EnterCriticalSection(0);

    /* Verificando se existe alguem esperando por sinal neste lock */
    for (i = 0; i < NPROCS; i++) {

      /* existe thread esperando sinal */
      if (_lock_data[lock].wait[i]) {

	if (i == HLRC_myself) {
	  /* thread bloqueada esta no processador local */
#ifdef BUSY_WAIT
	  HLRC_request_buffer[HLRC_myself][msg_type_index] = SIGNAL;	// acordar app
#else
	  pthread_cond_signal(&cond_);
#endif
#ifdef HLRC_DEBUG
	  fprintf(logfile,
		  "[%i] acordando thread do proc local\n",
		  HLRC_myself); fflush(logfile);
#endif

	} else {
	  /* Envio msg para o desbloquear thread remota */
#ifdef BUSY_WAIT
	  HLRC_scratch_buffer[HLRC_myself][i][msg_type_index] =
	    SIGNAL;
#else
	  HLRC_scratch_buffer[HLRC_myself][i][msg_type_index] =
	    NOSLEEP;
#endif
	  HLRC_scratch_buffer[HLRC_myself][i][msg_node_index] =
	    HLRC_myself;
	  HLRC_scratch_buffer[HLRC_myself][i][msg_item_index] =
	    lock;

#ifdef BUSY_WAIT
	  HLRC_SendMessage(i, i,
			   (msg_tmst1_start) * WORDSIZE, (char *)
			   &HLRC_scratch_buffer[HLRC_myself][i]
			   [msg_tmst1_start],
			   (NPROCS + MSG_HDR_LEN) * WORDSIZE,
			   REQUEST_MESG);
#else
	  HLRC_SendRequest(i, HLRC_myself,
			   (msg_tmst1_start) * WORDSIZE,
			   &HLRC_scratch_buffer[HLRC_myself][i]
			   [msg_tmst1_start],
			   (NPROCS + MSG_HDR_LEN) * sizeof(int),
			   NOSLEEP);
#endif

#ifdef HLRC_DEBUG
	  fprintf(logfile,
		  "[%i] HLRC_SendMessage (acordando thread remota) dest:%i\n",
		  HLRC_myself, i);
	  fflush(logfile);
#endif

	}

	/* retirando thread da fila de espera */
	_lock_data[lock].wait[i] = 0;

	break;
      }

    }
    HLRC_LeaveCriticalSection(0);
  } else {
    HLRC_EnterCriticalSection(0);
    /* Envio requisicao para o gerente do lock */
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_type_index] =
      SIGNAL;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_node_index] =
      HLRC_myself;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_item_index] = lock;

    HLRC_SendRequest(lmanager, HLRC_myself,
		     (msg_tmst1_start) * WORDSIZE,
		     &HLRC_scratch_buffer[HLRC_myself][lmanager]
		     [msg_tmst1_start],
		     (NPROCS + MSG_HDR_LEN) * sizeof(int), SIGNAL);
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] HLRC_SendRequest (SIGNAL) dest:%i\n",
	    HLRC_myself, lmanager);
    fflush(logfile);
#endif
    HLRC_LeaveCriticalSection(0);
  }
}

void HLRC_lock_cond_broadcast(unsigned lock, unsigned cond)
{
  int lmanager = lock % HLRC_nprocs;
  int i;

#ifdef HLRC_DEBUG
  fprintf(logfile, "[%i] HLRC_lock_cond_broadcast lock:%i\n",
	  HLRC_myself, lock);
  fflush(logfile);
#endif

  /* Eu sou o gerente */
  if (lmanager == HLRC_myself) {

    //    fprintf(logfile,"[%i] Eu sou o gerente\n", HLRC_myself); fflush(stdout);
    HLRC_EnterCriticalSection(0);
    /* Verificando se existe alguem esperando por sinal neste lock */
    for (i = 0; i < NPROCS; i++) {

      /* existe thread esperando sinal */
      if (_lock_data[lock].wait[i]) {

	if (i == HLRC_myself) {
	  /* thread bloqueada esta no processador local */
#ifdef BUSY_WAIT
	  HLRC_request_buffer[HLRC_myself][msg_type_index] = SIGNAL;	// acordar app
#else
	  pthread_cond_signal(&cond_);
#endif
#ifdef HLRC_DEBUG
	  fprintf(logfile,
		  "[%i] acordando thread do proc local\n",
		  HLRC_myself); fflush(logfile);
#endif
	} else {
	  /* Envio msg para o desbloquear thread remota */
#ifdef BUSY_WAIT
	  HLRC_scratch_buffer[HLRC_myself][i][msg_type_index] =
	    SIGNAL;
#else
	  HLRC_scratch_buffer[HLRC_myself][i][msg_type_index] =
	    NOSLEEP;
#endif
	  HLRC_scratch_buffer[HLRC_myself][i][msg_node_index] =
	    HLRC_myself;
	  HLRC_scratch_buffer[HLRC_myself][i][msg_item_index] =
	    lock;

#ifdef BUSY_WAIT
	  HLRC_SendMessage(i, i,
			   (msg_tmst1_start) * WORDSIZE, (char *)
			   &HLRC_scratch_buffer[HLRC_myself][i]
			   [msg_tmst1_start],
			   (NPROCS + MSG_HDR_LEN) * WORDSIZE,
			   REQUEST_MESG, 0);
#else
	  HLRC_SendRequest(i, HLRC_myself,
			   (msg_tmst1_start) * WORDSIZE,
			   &HLRC_scratch_buffer[HLRC_myself][i]
			   [msg_tmst1_start],
			   (NPROCS + MSG_HDR_LEN) * sizeof(int),
			   NOSLEEP);
#endif
#ifdef HLRC_DEBUG
	  fprintf(logfile,
		  "[%i] HLRC_SendMessage (acordando thread remota) dest:%i\n",
		  HLRC_myself, i);
	  fflush(logfile);
#endif

	}

	/* retirando thread da fila de espera */

	_lock_data[lock].wait[i] = 0;


      }

    }
    HLRC_LeaveCriticalSection(0);
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] Saindo HLRC_lock_cond_broadcast : %i\n",
	    HLRC_myself, lmanager);
    fflush(logfile);
#endif
  } else {
    HLRC_EnterCriticalSection(0);
    /* Envio requisicao para o gerente do lock */
#ifdef HLRC_DEBUG
    fprintf(logfile, "[%i] HLRC_SendRequest (BROADCAST) dest:%i\n",
	    HLRC_myself, lmanager);
    fflush(logfile);
#endif
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_type_index] =
      BROADCAST;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_node_index] =
      HLRC_myself;
    HLRC_scratch_buffer[HLRC_myself][lmanager][msg_item_index] = lock;

    HLRC_SendRequest(lmanager, HLRC_myself,
		     (msg_tmst1_start) * WORDSIZE,
		     &HLRC_scratch_buffer[HLRC_myself][lmanager]
		     [msg_tmst1_start],
		     (NPROCS + MSG_HDR_LEN) * sizeof(int), BROADCAST);
    HLRC_LeaveCriticalSection(0);
  }
}

/*
  Page fault handler:
  All pages have a home node, which is responsible for
  maintaining the shared memory page. Initially, all pages 
  are assigned default homes - modulo (HLRC_nprocs) on the 
  page number. The first node to access a page becomes the 
  home of the page. Initially, requests are sent to the 
  default home, which assigns a home on the first access
  and all subsequent requests are forwarded to the newly
  assigned home.
*/

void
HLRC_PageHandler5 (int sig, siginfo_t *sip, void *uc){
  HLRC_ErrPrint("Sinal 5 recebido!\n");
}

void
HLRC_PageHandler (int sig, siginfo_t *sip, void *uc) {
  /*  char pid[10];
      int mypid,npid, pdb_flag=1;*/
  register int i, j;
  register int relpage;
  unsigned pageno;
  unsigned address;
  unsigned error;
  unsigned home;
  unsigned long long time1, time2, time3, time4;
#ifdef NO_SWAP_READONLY_PAGE
  unsigned page_not_present = 0;
#endif
  //  HLRC_ErrPrint("Inicio\n");
  GETTIME (time1);

#if 0
  curr_cpu = sysmp (MP_CURPROC);
  if (curr_cpu != app_cpu)
    {
      fprintf (logfile, "Switched CPU\n");
      fflush (logfile);
      exit (1);
    }
#endif

  HLRC_stat.page_faults++;

  /*
    Faulting error type(READ/WRITE)
  */
  //  error = (scs.err & 2); //original
  //  address = (int) scs.cr2; //original

  error = (((ucontext_t *)uc)->uc_mcontext.gregs[13] & 2) >> 1;
  //address = (caddr_t)sip->si_addr; Anderson
  address = (unsigned)sip->si_addr;

  /*
    Faulting page number
  */

  pageno = address / pagesize;


  //if (address == 0x402c4000) while (1);

  if ((pageno < _first_shared_page) || (pageno > _last_shared_page))
    {

      /*        mypid = getpid();
		bzero(pid,10);
		sprintf(pid,"%d",mypid);
	
		if (fork() == 0)
		execlp("/usr/bin/xterm", "xterm", "-sb", "-rv", "/usr/bin/gdb", "/home/cjava/marcelo+lobosco+anderson_25_01_2006/control/build/linux-i586/bin/java", pid, NULL);
	
		while (pdb_flag);*/
      
      fprintf (logfile, "[%d] Wrong page: %d(%x), error: %d\n",
	       (int)pthread_self (), pageno, address, error);
      fflush (logfile);
      HLRC_ErrPrint ("Wrong page: %d(%x), error: %d\n", pageno, address,
		     error);
      HLRC_ErrPrint ("PN:%d FSP:%d LSP:%d\n", pageno, _first_shared_page, _last_shared_page);	//lobosco
      while (1);
      abort ();
    }
  relpage = pageno - _first_shared_page;

#ifdef HLRC_DEBUG
  fprintf (logfile, "[%d]: Page fault handler: %x => %d(%d), error: %d\n",
	   (int)pthread_self (), address, relpage, _per_page[relpage].home, error);
  fflush (logfile);

  //  if(address==0x411f0928) while (1);

#endif

  home = relpage % HLRC_nprocs;

#endif

  /*  int debug=1;
      if (HLRC_myself == 1 && relpage == 6441) while (debug);*/

  /*
    Write fault
  */

  if (error)
    //  if (error) //original
    {
      //HLRC_ErrPrint("Erro\n");
#ifdef HLRC_DEBUG
      fprintf (logfile, "WRITE FAULT\n");
      fflush (logfile);
#endif

      HLRC_EnterCriticalSection (0);

#ifdef NO_SWAP_READONLY_PAGE
      error =
	mprotect ((void *) (pageno << logpagesize), pagesize,
		  QUERY_PAGE_PROT);
      if (error == PAGE_NOT_PRESENT || error == -PAGE_NOT_PRESENT)
	{
	  page_not_present = 1;
	}
#endif

      if (error =
	  mprotect ((void *) (pageno << logpagesize), pagesize, HLRC_PROT_RW))
	{
	  fprintf (logfile, "1: mprotect error: page %d\n", relpage);
	  fflush (logfile);
	  perror ("mprotect error");
	  exit (1);
	}

      HLRC_LeaveCriticalSection (0);

      if (_per_page[relpage].home == -1)
	{
	  
	  /*
	    Home not yet known
	  */
 
	  if (home == HLRC_myself)
	    {
	      HLRC_EnterCriticalSection (0);

	      if (_per_page[relpage].home == -1)
		{
		  /*
		    If home is not yet assigned, the 
		    default home becomes the home
		  */
		  _per_page[relpage].home = HLRC_myself;
		  HLRC_stat.homes++;
#ifdef HLRC_DEBUG
		  fprintf (logfile, "I am home for page: %d\n", relpage);
		  fflush (logfile);
#endif
		}
	      HLRC_LeaveCriticalSection (0);
	    }
	  else
	    {
	      /*
		Contact default home for the page
	      */
	      
	      GETTIME (time2);
	      HLRC_stat.time_over += time2 - time1;

	      HLRC_stat.page_misses++;
#ifdef HLRC_DEBUG
	      fprintf (logfile, "Sending request for page: %d my_stamp: %p\n", relpage, _per_page[relpage].my_stamp);
	      fflush (logfile);
#endif
	      HLRC_SendRequest (home, HLRC_myself,
				(msg_tmst1_start) * WORDSIZE,
				_per_page[relpage].my_stamp,
				(NPROCS + MSG_HDR_LEN) * sizeof (int), PAGE);
	      HLRC_Recv (PAGE, (volatile int *)&_per_page[relpage].home, NPROCS + 1, relpage);

	      GETTIME (time1);
	      HLRC_stat.time_comm += time1 - time2;

	    }
	}
      else
	{
	  /*
	    Home already known
	  */
	  
	  if (_per_page[relpage].state < 0)
	    {
	      /*
		Only at the home, can the state be negative
	      */
	      _per_page[relpage].state = -_per_page[relpage].state;
	      if (_per_page[relpage].home == HLRC_myself)
		{
		  for (i = 0; i < HLRC_nprocs; i++)
		    {
		      GETTIME (time3);
		      while (_per_page[relpage].my_stamp[i] >
			     _home_flush[i][relpage]);
		      GETTIME (time4);
		      HLRC_stat.time_spinning += (time4 - time3);
		    }
		}
	      else
		{
		  fprintf (logfile,
			   "Error: Page(%d) state negative at non-home node\n",
			   relpage);
		  fflush (logfile);
		  HLRC_ErrPrint
		    ("Error: Page(%d) state negative at non-home node\n",
		     relpage);
		  exit (1);
		}
#ifdef NO_SWAP_READONLY_PAGE
	    }
	  else if ((_per_page[relpage].state == 0) || page_not_present)
	    {

	      if (page_not_present)
		{
		  _per_page[relpage].state = 0;
		}
#else
	    }
	  else if (_per_page[relpage].state == 0)
	    {
#endif

	      GETTIME (time2);
	      HLRC_stat.time_over += (time2 - time1);
	      HLRC_stat.page_misses++;
	      HLRC_SendRequest (_per_page[relpage].home,
				HLRC_myself, (msg_tmst1_start) * WORDSIZE,
				_per_page[relpage].my_stamp,
				(NPROCS + MSG_HDR_LEN) * sizeof (int), PAGE);
	      HLRC_Recv (PAGE, &_per_page[relpage].my_stamp[0],
			 NPROCS, relpage);
	      GETTIME (time1);
	      HLRC_stat.time_comm += time1 - time2;

	    }
	}

      if (_per_page[relpage].state < HLRC_PROT_RW)
	{
	  _update_list[_index_update++] = relpage;
	  if (_index_update == NUPDATES)
	    {
	      fprintf (logfile, "Update list overflow\n");
	      fprintf (logfile, "Try increasing NUPDATES(currently %d)\n",
		       NUPDATES);
	      fflush (logfile);
	      HLRC_ErrPrint ("Update list overflow\n");
	      HLRC_ErrPrint ("Try increasing NUPDATES(currently %d)\n",
			     NUPDATES);
	      exit (1);
	    }
	  _per_page[relpage].state = HLRC_PROT_RW;

	  if (_per_page[relpage].home != HLRC_myself)
	    {
	      _per_page[relpage].twin = HLRC_AllocTwin ();

	      if (_per_page[relpage].twin == NULL) {
                _per_page[relpage].twin = (int *) _AMALLOC (pagesize, pagesize);	
#ifdef HLRC_DEBUG

		fprintf (logfile, "[%d] twin[%i] == %p\n", (int)pthread_self (), relpage, _per_page[relpage].twin);
		fflush (logfile);
#endif
	      }
	      bcopy ((const void *)(pageno << logpagesize), _per_page[relpage].twin,
		     _per_page[relpage].size);
	    }

	}

    }
  else
    {
      /*
	Read fault
      */
#ifdef HLRC_DEBUG
      fprintf (logfile, "READ FAULT\n");
      fflush (logfile);
#endif
      if (_per_page[relpage].home == -1)
	{
	  /*
	    Home not yet known
	  */
	  
	  if (error =
	      mprotect ((void *) (pageno << logpagesize), pagesize,
			HLRC_PROT_READ))
	    {
	      fprintf (logfile, "2: mprotect error: page %d\n", relpage);
	      fflush (logfile);
	      perror ("mprotect error");
	      exit (1);
	    }

	  _per_page[relpage].state = 1;
 
	  if (home == HLRC_myself)
	    {
	      /*
		I am the default home
	      */
	      HLRC_EnterCriticalSection (0);

	      if (_per_page[relpage].home == -1)
		{
		  /*
		    Default home becomes home
		  */
#ifdef HLRC_DEBUG
		  fprintf (logfile, "I am home for page: %d\n", relpage);
		  fflush (logfile);
#endif
		  _per_page[relpage].home = HLRC_myself;
		  HLRC_stat.homes++;

		  HLRC_LeaveCriticalSection (0);
		}
	      else
		{

		  HLRC_LeaveCriticalSection (0);
		}
	    }
	  else
	    {
	      /*
		I am NOT the default home. Contact
		the default home for the page.
	      */

	      GETTIME (time2);
	      HLRC_stat.time_over += time2 - time1;

	      HLRC_stat.page_misses++;

#if !defined(PROT_HANDLE) && defined(COPY_ON_RECV)
	      if (error = mprotect ((void *) (pageno << logpagesize),
				    pagesize, HLRC_PROT_RW))
		{
		  fprintf (logfile, "4a: mprotect error: page %d\n", relpage);
		  fflush (logfile);
		  perror ("mprotect error");
		  exit (1);
		}
#endif
  
	      HLRC_SendRequest (home, HLRC_myself,
				(msg_tmst1_start) * WORDSIZE,
				_per_page[relpage].my_stamp,
				(NPROCS + MSG_HDR_LEN) * sizeof (int), PAGE);
	      HLRC_Recv (PAGE, (volatile int *)&_per_page[relpage].home, NPROCS + 1, relpage);
	      
#if !defined(PROT_HANDLE) && defined(COPY_ON_RECV)
	      
	      if (error = mprotect ((void *) (pageno << logpagesize),
				    pagesize, HLRC_PROT_READ))
		{
		  fprintf (logfile, "4a: mprotect error: page %d\n", relpage);
		  fflush (logfile);
		  perror ("mprotect error");
		  exit (1);
		}
#endif

	      GETTIME (time1);
	      HLRC_stat.time_comm += time1 - time2;
	    }

	}
      else if (_per_page[relpage].state < 0)
	{

	  /*
	    Only home can have negative state
	  */
	  if (_per_page[relpage].home == HLRC_myself)
	    {
	      for (i = 0; i < HLRC_nprocs; i++)
		{
		  GETTIME (time3);
		  while (_per_page[relpage].my_stamp[i] >
			 _home_flush[i][relpage]);
		  GETTIME (time4);
		  HLRC_stat.time_spinning += time4 - time3;
		}
	      _per_page[relpage].state = -_per_page[relpage].state;
	      if (_per_page[relpage].state < HLRC_EXCL_STATE)
		{

		  if (error =
		      mprotect ((void *) ((relpage + _first_shared_page) <<
					  logpagesize), pagesize,
				_per_page[relpage].state))
		    {
		      fprintf (logfile, "5: mprotect error: page %d\n",
			       relpage);
		      fflush (logfile);
		      perror ("mprotect error");
		      exit (1);
		    }
		}
	      else
		{
		  if (error =
		      mprotect ((void *) ((relpage + _first_shared_page) <<
					  logpagesize), pagesize,
				HLRC_PROT_RW))
		    {
		      fprintf (logfile, "6: mprotect error: page %d\n",
			       relpage);
		      fflush (logfile);
		      perror ("mprotect error");
		      exit (1);
		    }
		}
	    }
	  else
	    {
	      fprintf (logfile,
		       "Error: Page(%d) state negative at non-home node\n",
		       relpage);
	      fflush (logfile);
	      HLRC_ErrPrint
		("Error: Page(%d) state negative at non-home node\n",
		 relpage);
	      exit (1);
	    }

	}
      else
	{
#ifdef NO_SWAP_READONLY_PAGE
	  error =
	    mprotect ((void *) (pageno << logpagesize), pagesize,
		      QUERY_PAGE_PROT);
	  if (error == PAGE_NOT_PRESENT || error == -PAGE_NOT_PRESENT)
	    {
	      page_not_present = 1;
	    }
#endif

	  /*
	    Now, we know the home and this is not the home.
	    Send a PAGE request to the home.
	  */
#ifdef NO_SWAP_READONLY_PAGE
	  if ((_per_page[relpage].state == 0) || page_not_present)
	    {

	      _per_page[relpage].state = 0;
#else
	      if (_per_page[relpage].state == 0)
		{
#endif

#if !defined(PROT_HANDLE) && defined(COPY_ON_RECV)
		  if (error = mprotect ((void *) (pageno << logpagesize),
					pagesize, HLRC_PROT_RW))
		    {
		      fprintf (logfile, "7: mprotect error: page %d\n", relpage);
		      fflush (logfile);
		      perror ("mprotect error");
		      exit (1);
		    }
#else
		  if (error = mprotect ((void *) (pageno << logpagesize),
					pagesize, HLRC_PROT_READ))
		    {
		      fprintf (logfile, "7: mprotect error: page %d\n", relpage);
		      fflush (logfile);
		      perror ("mprotect error");
		      exit (1);
		    }
#endif

		  _per_page[relpage].state = 1;

		  GETTIME (time2);
		  HLRC_stat.time_over += (time2 - time1);

		  HLRC_stat.page_misses++;

		  HLRC_SendRequest (_per_page[relpage].home,
				    HLRC_myself, (msg_tmst1_start) * WORDSIZE,
				    _per_page[relpage].my_stamp,
				    (NPROCS + MSG_HDR_LEN) * sizeof (int), PAGE);
		  HLRC_Recv (PAGE, &_per_page[relpage].my_stamp[0],
			     NPROCS, relpage);

#if !defined(PROT_HANDLE) && defined(COPY_ON_RECV)
		  if (error = mprotect ((void *) (pageno << logpagesize),
					pagesize, HLRC_PROT_READ))
		    {
		      fprintf (logfile, "7: mprotect error: page %d\n", relpage);
		      fflush (logfile);
		      perror ("mprotect error");
		      exit (1);
		    }
#endif

		  GETTIME (time1);
		  HLRC_stat.time_comm += (time1 - time2);

		}
	      else
		{
		  if (error =
		      mprotect ((void *) (pageno << logpagesize), pagesize,
				HLRC_PROT_READ))
		    {
		      fprintf (logfile, "9: mprotect error: page %d\n", relpage);
		      fflush (logfile);
		      perror ("mprotect error");
		      exit (1);
		    }
		  _per_page[relpage].state = 1;
		}
	    }
	}

      GETTIME (time2);
      HLRC_stat.time_over += time2 - time1; 

#ifdef HLRC_DEBUG
      fprintf (logfile, "Page fault handler done\n");
      fflush (logfile);
#endif
      //  HLRC_ErrPrint("Fim\n");
    }
